2024-12-05 12:13:10,778 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 12:13:10,787 main DEBUG Took 0.008295 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 12:13:10,788 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 12:13:10,788 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 12:13:10,789 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 12:13:10,790 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,796 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 12:13:10,807 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,809 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,809 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,810 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,811 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,812 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,812 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,812 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,812 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,813 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,813 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,813 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,814 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,815 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,816 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:13:10,816 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,816 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 12:13:10,818 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:13:10,819 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 12:13:10,821 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 12:13:10,821 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 12:13:10,822 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 12:13:10,823 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 12:13:10,830 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 12:13:10,832 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 12:13:10,834 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 12:13:10,834 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 12:13:10,835 main DEBUG createAppenders(={Console}) 2024-12-05 12:13:10,835 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-05 12:13:10,836 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 12:13:10,836 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-05 12:13:10,836 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 12:13:10,836 main DEBUG OutputStream closed 2024-12-05 12:13:10,837 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 12:13:10,837 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 12:13:10,837 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-05 12:13:10,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 12:13:10,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 12:13:10,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 12:13:10,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 12:13:10,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 12:13:10,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 12:13:10,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 12:13:10,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 12:13:10,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 12:13:10,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 12:13:10,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 12:13:10,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 12:13:10,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 12:13:10,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 12:13:10,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 12:13:10,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 12:13:10,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 12:13:10,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 12:13:10,908 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 12:13:10,909 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-05 12:13:10,909 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 12:13:10,910 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-05T12:13:11,105 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3 2024-12-05 12:13:11,108 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 12:13:11,108 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T12:13:11,116 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-05T12:13:11,134 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T12:13:11,154 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/cluster_9f95a8c6-304c-dd62-ec56-918ba890d196, deleteOnExit=true 2024-12-05T12:13:11,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T12:13:11,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/test.cache.data in system properties and HBase conf 2024-12-05T12:13:11,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T12:13:11,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/hadoop.log.dir in system properties and HBase conf 2024-12-05T12:13:11,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T12:13:11,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T12:13:11,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T12:13:11,238 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T12:13:11,324 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:13:11,328 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T12:13:11,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T12:13:11,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T12:13:11,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T12:13:11,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T12:13:11,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T12:13:11,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T12:13:11,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T12:13:11,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T12:13:11,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/nfs.dump.dir in system properties and HBase conf 2024-12-05T12:13:11,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/java.io.tmpdir in system properties and HBase conf 2024-12-05T12:13:11,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T12:13:11,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T12:13:11,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T12:13:12,291 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T12:13:12,355 INFO [Time-limited test {}] log.Log(170): Logging initialized @2171ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T12:13:12,415 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T12:13:12,470 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T12:13:12,493 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T12:13:12,494 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T12:13:12,495 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T12:13:12,508 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T12:13:12,510 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/hadoop.log.dir/,AVAILABLE} 2024-12-05T12:13:12,511 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T12:13:12,697 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/java.io.tmpdir/jetty-localhost-32829-hadoop-hdfs-3_4_1-tests_jar-_-any-11674868937669494631/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T12:13:12,708 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:32829} 2024-12-05T12:13:12,708 INFO [Time-limited test {}] server.Server(415): Started @2525ms 2024-12-05T12:13:13,179 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T12:13:13,187 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T12:13:13,188 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T12:13:13,188 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T12:13:13,189 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T12:13:13,190 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/hadoop.log.dir/,AVAILABLE} 2024-12-05T12:13:13,190 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T12:13:13,284 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/java.io.tmpdir/jetty-localhost-33197-hadoop-hdfs-3_4_1-tests_jar-_-any-14341986859328782066/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T12:13:13,285 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:33197} 2024-12-05T12:13:13,285 INFO [Time-limited test {}] server.Server(415): Started @3101ms 2024-12-05T12:13:13,333 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T12:13:14,347 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/cluster_9f95a8c6-304c-dd62-ec56-918ba890d196/data/data1/current/BP-1803855376-172.17.0.3-1733400791885/current, will proceed with Du for space computation calculation, 2024-12-05T12:13:14,347 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/cluster_9f95a8c6-304c-dd62-ec56-918ba890d196/data/data2/current/BP-1803855376-172.17.0.3-1733400791885/current, will proceed with Du for space computation calculation, 2024-12-05T12:13:14,397 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T12:13:14,451 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x292e3f45b8b1c1a3 with lease ID 0xeac124e9ca128e53: Processing first storage report for DS-4043a30f-b058-46e5-afb3-e568bf768ae5 from datanode DatanodeRegistration(127.0.0.1:32993, datanodeUuid=8d221f7e-d96d-4699-8801-d839600c5300, infoPort=39773, infoSecurePort=0, ipcPort=39175, storageInfo=lv=-57;cid=testClusterID;nsid=309051644;c=1733400791885) 2024-12-05T12:13:14,452 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x292e3f45b8b1c1a3 with lease ID 0xeac124e9ca128e53: from storage DS-4043a30f-b058-46e5-afb3-e568bf768ae5 node DatanodeRegistration(127.0.0.1:32993, datanodeUuid=8d221f7e-d96d-4699-8801-d839600c5300, infoPort=39773, infoSecurePort=0, ipcPort=39175, storageInfo=lv=-57;cid=testClusterID;nsid=309051644;c=1733400791885), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T12:13:14,453 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x292e3f45b8b1c1a3 with lease ID 0xeac124e9ca128e53: Processing first storage report for DS-efe279af-2f96-4cb3-b063-8310606ed9be from datanode DatanodeRegistration(127.0.0.1:32993, datanodeUuid=8d221f7e-d96d-4699-8801-d839600c5300, infoPort=39773, infoSecurePort=0, ipcPort=39175, storageInfo=lv=-57;cid=testClusterID;nsid=309051644;c=1733400791885) 2024-12-05T12:13:14,453 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x292e3f45b8b1c1a3 with lease ID 0xeac124e9ca128e53: from storage DS-efe279af-2f96-4cb3-b063-8310606ed9be node DatanodeRegistration(127.0.0.1:32993, datanodeUuid=8d221f7e-d96d-4699-8801-d839600c5300, infoPort=39773, infoSecurePort=0, ipcPort=39175, storageInfo=lv=-57;cid=testClusterID;nsid=309051644;c=1733400791885), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T12:13:14,519 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3 2024-12-05T12:13:14,594 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/cluster_9f95a8c6-304c-dd62-ec56-918ba890d196/zookeeper_0, clientPort=59678, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/cluster_9f95a8c6-304c-dd62-ec56-918ba890d196/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/cluster_9f95a8c6-304c-dd62-ec56-918ba890d196/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T12:13:14,606 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59678 2024-12-05T12:13:14,618 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T12:13:14,621 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T12:13:14,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741825_1001 (size=7) 2024-12-05T12:13:15,279 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e with version=8 2024-12-05T12:13:15,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/hbase-staging 2024-12-05T12:13:15,377 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T12:13:15,672 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9ea9abad0e9b:0 server-side Connection retries=45 2024-12-05T12:13:15,686 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:13:15,686 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T12:13:15,692 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T12:13:15,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:13:15,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T12:13:15,840 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T12:13:15,899 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T12:13:15,910 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T12:13:15,915 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T12:13:15,944 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2828 (auto-detected) 2024-12-05T12:13:15,945 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-05T12:13:15,965 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34103 2024-12-05T12:13:15,983 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34103 connecting to ZooKeeper ensemble=127.0.0.1:59678 2024-12-05T12:13:16,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:341030x0, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T12:13:16,075 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34103-0x101a696db740000 connected 2024-12-05T12:13:16,153 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T12:13:16,156 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T12:13:16,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:13:16,171 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e, hbase.cluster.distributed=false 2024-12-05T12:13:16,200 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T12:13:16,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34103 2024-12-05T12:13:16,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34103 2024-12-05T12:13:16,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34103 2024-12-05T12:13:16,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34103 2024-12-05T12:13:16,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34103 2024-12-05T12:13:16,312 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ea9abad0e9b:0 server-side Connection retries=45 2024-12-05T12:13:16,314 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:13:16,314 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T12:13:16,314 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T12:13:16,314 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:13:16,314 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T12:13:16,317 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T12:13:16,320 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T12:13:16,321 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44759 2024-12-05T12:13:16,324 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44759 connecting to ZooKeeper ensemble=127.0.0.1:59678 2024-12-05T12:13:16,325 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T12:13:16,329 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T12:13:16,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447590x0, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T12:13:16,343 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:447590x0, quorum=127.0.0.1:59678, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:13:16,344 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44759-0x101a696db740001 connected 2024-12-05T12:13:16,348 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T12:13:16,357 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T12:13:16,360 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T12:13:16,366 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T12:13:16,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44759 2024-12-05T12:13:16,367 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44759 2024-12-05T12:13:16,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44759 2024-12-05T12:13:16,371 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44759 2024-12-05T12:13:16,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44759 2024-12-05T12:13:16,390 DEBUG [M:0;9ea9abad0e9b:34103 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9ea9abad0e9b:34103 2024-12-05T12:13:16,391 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:16,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:13:16,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:13:16,409 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:16,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:16,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T12:13:16,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:16,440 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T12:13:16,441 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9ea9abad0e9b,34103,1733400795471 from backup master directory 2024-12-05T12:13:16,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:16,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:13:16,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:13:16,452 WARN [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T12:13:16,452 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:16,454 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T12:13:16,455 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T12:13:16,518 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/hbase.id] with ID: a3fb9d9d-e54c-41a0-a632-14306d0e2ea2 2024-12-05T12:13:16,519 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/.tmp/hbase.id 2024-12-05T12:13:16,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741826_1002 (size=42) 2024-12-05T12:13:16,935 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/.tmp/hbase.id]:[hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/hbase.id] 2024-12-05T12:13:16,981 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T12:13:16,987 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T12:13:17,004 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-05T12:13:17,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:17,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:17,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741827_1003 (size=196) 2024-12-05T12:13:17,485 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:13:17,487 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T12:13:17,500 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:17,504 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:13:17,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741828_1004 (size=1189) 2024-12-05T12:13:17,955 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store 2024-12-05T12:13:17,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741829_1005 (size=34) 2024-12-05T12:13:18,383 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T12:13:18,387 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:13:18,388 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T12:13:18,388 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:13:18,388 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:13:18,390 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T12:13:18,390 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:13:18,390 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:13:18,392 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733400798388Disabling compacts and flushes for region at 1733400798388Disabling writes for close at 1733400798390 (+2 ms)Writing region close event to WAL at 1733400798390Closed at 1733400798390 2024-12-05T12:13:18,394 WARN [master/9ea9abad0e9b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/.initializing 2024-12-05T12:13:18,394 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/WALs/9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:18,402 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T12:13:18,417 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ea9abad0e9b%2C34103%2C1733400795471, suffix=, logDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/WALs/9ea9abad0e9b,34103,1733400795471, archiveDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/oldWALs, maxLogs=10 2024-12-05T12:13:18,438 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/WALs/9ea9abad0e9b,34103,1733400795471/9ea9abad0e9b%2C34103%2C1733400795471.1733400798421, exclude list is [], retry=0 2024-12-05T12:13:18,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32993,DS-4043a30f-b058-46e5-afb3-e568bf768ae5,DISK] 2024-12-05T12:13:18,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T12:13:18,489 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/WALs/9ea9abad0e9b,34103,1733400795471/9ea9abad0e9b%2C34103%2C1733400795471.1733400798421 2024-12-05T12:13:18,490 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39773:39773)] 2024-12-05T12:13:18,490 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:13:18,491 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:13:18,494 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,495 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T12:13:18,557 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:18,559 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:13:18,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,563 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T12:13:18,563 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:18,564 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:18,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,567 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T12:13:18,568 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:18,568 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:18,569 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,571 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T12:13:18,572 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:18,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:18,573 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,576 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,577 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,582 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,583 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,587 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T12:13:18,590 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:13:18,595 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:13:18,596 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69365461, jitterRate=0.03362591564655304}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T12:13:18,600 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733400798506Initializing all the Stores at 1733400798508 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400798508Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400798509 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400798509Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400798509Cleaning up temporary data from old regions at 1733400798583 (+74 ms)Region opened successfully at 1733400798600 (+17 ms) 2024-12-05T12:13:18,602 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T12:13:18,635 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16bee8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ea9abad0e9b/172.17.0.3:0 2024-12-05T12:13:18,662 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T12:13:18,672 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T12:13:18,672 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T12:13:18,674 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T12:13:18,676 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T12:13:18,680 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T12:13:18,680 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T12:13:18,704 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T12:13:18,713 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T12:13:18,809 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T12:13:18,815 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T12:13:18,817 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T12:13:18,825 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T12:13:18,827 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T12:13:18,831 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T12:13:18,842 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T12:13:18,844 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T12:13:18,850 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T12:13:18,872 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T12:13:18,880 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T12:13:18,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T12:13:18,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T12:13:18,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:18,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:18,896 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9ea9abad0e9b,34103,1733400795471, sessionid=0x101a696db740000, setting cluster-up flag (Was=false) 2024-12-05T12:13:18,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:18,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:18,951 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T12:13:18,953 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:19,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:19,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:19,051 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T12:13:19,054 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:19,063 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T12:13:19,080 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(746): ClusterId : a3fb9d9d-e54c-41a0-a632-14306d0e2ea2 2024-12-05T12:13:19,082 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T12:13:19,094 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T12:13:19,094 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T12:13:19,102 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T12:13:19,103 DEBUG [RS:0;9ea9abad0e9b:44759 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3757ec3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ea9abad0e9b/172.17.0.3:0 2024-12-05T12:13:19,116 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9ea9abad0e9b:44759 2024-12-05T12:13:19,119 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T12:13:19,119 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T12:13:19,119 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T12:13:19,122 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ea9abad0e9b,34103,1733400795471 with port=44759, startcode=1733400796278 2024-12-05T12:13:19,135 DEBUG [RS:0;9ea9abad0e9b:44759 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T12:13:19,139 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T12:13:19,149 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T12:13:19,158 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T12:13:19,166 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46153, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T12:13:19,166 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9ea9abad0e9b,34103,1733400795471 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T12:13:19,171 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34103 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:19,176 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9ea9abad0e9b:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:13:19,176 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9ea9abad0e9b:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:13:19,176 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9ea9abad0e9b:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:13:19,177 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9ea9abad0e9b:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:13:19,177 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9ea9abad0e9b:0, corePoolSize=10, maxPoolSize=10 2024-12-05T12:13:19,177 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,178 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9ea9abad0e9b:0, corePoolSize=2, maxPoolSize=2 2024-12-05T12:13:19,178 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,182 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733400829182 2024-12-05T12:13:19,184 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T12:13:19,184 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T12:13:19,186 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T12:13:19,186 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T12:13:19,188 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T12:13:19,189 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T12:13:19,189 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T12:13:19,189 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T12:13:19,190 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,194 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T12:13:19,194 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:19,194 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T12:13:19,195 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T12:13:19,196 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T12:13:19,198 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T12:13:19,198 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T12:13:19,201 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9ea9abad0e9b:0:becomeActiveMaster-HFileCleaner.large.0-1733400799200,5,FailOnTimeoutGroup] 2024-12-05T12:13:19,201 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-05T12:13:19,201 WARN [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-05T12:13:19,203 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9ea9abad0e9b:0:becomeActiveMaster-HFileCleaner.small.0-1733400799201,5,FailOnTimeoutGroup] 2024-12-05T12:13:19,203 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,203 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T12:13:19,204 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,204 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741831_1007 (size=1321) 2024-12-05T12:13:19,302 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ea9abad0e9b,34103,1733400795471 with port=44759, startcode=1733400796278 2024-12-05T12:13:19,305 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34103 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:19,307 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34103 {}] master.ServerManager(517): Registering regionserver=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:19,317 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e 2024-12-05T12:13:19,317 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46811 2024-12-05T12:13:19,317 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T12:13:19,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T12:13:19,326 DEBUG [RS:0;9ea9abad0e9b:44759 {}] zookeeper.ZKUtil(111): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:19,326 WARN [RS:0;9ea9abad0e9b:44759 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T12:13:19,326 INFO [RS:0;9ea9abad0e9b:44759 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:13:19,327 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/WALs/9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:19,329 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ea9abad0e9b,44759,1733400796278] 2024-12-05T12:13:19,353 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T12:13:19,364 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T12:13:19,369 INFO [RS:0;9ea9abad0e9b:44759 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T12:13:19,369 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,370 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T12:13:19,374 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T12:13:19,375 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,376 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,376 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,376 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,376 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,376 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,376 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ea9abad0e9b:0, corePoolSize=2, maxPoolSize=2 2024-12-05T12:13:19,376 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,376 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,377 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,377 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,377 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,377 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ea9abad0e9b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:13:19,377 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ea9abad0e9b:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:13:19,377 DEBUG [RS:0;9ea9abad0e9b:44759 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:13:19,378 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,379 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,379 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,379 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,379 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,379 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ea9abad0e9b,44759,1733400796278-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T12:13:19,395 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T12:13:19,397 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ea9abad0e9b,44759,1733400796278-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,397 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,397 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.Replication(171): 9ea9abad0e9b,44759,1733400796278 started 2024-12-05T12:13:19,412 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:19,412 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1482): Serving as 9ea9abad0e9b,44759,1733400796278, RpcServer on 9ea9abad0e9b/172.17.0.3:44759, sessionid=0x101a696db740001 2024-12-05T12:13:19,413 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T12:13:19,413 DEBUG [RS:0;9ea9abad0e9b:44759 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:19,413 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ea9abad0e9b,44759,1733400796278' 2024-12-05T12:13:19,413 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T12:13:19,414 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T12:13:19,415 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T12:13:19,415 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T12:13:19,415 DEBUG [RS:0;9ea9abad0e9b:44759 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:19,415 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ea9abad0e9b,44759,1733400796278' 2024-12-05T12:13:19,415 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T12:13:19,416 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T12:13:19,417 DEBUG [RS:0;9ea9abad0e9b:44759 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T12:13:19,417 INFO [RS:0;9ea9abad0e9b:44759 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T12:13:19,417 INFO [RS:0;9ea9abad0e9b:44759 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T12:13:19,527 INFO [RS:0;9ea9abad0e9b:44759 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T12:13:19,530 INFO [RS:0;9ea9abad0e9b:44759 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ea9abad0e9b%2C44759%2C1733400796278, suffix=, logDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/WALs/9ea9abad0e9b,44759,1733400796278, archiveDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/oldWALs, maxLogs=32 2024-12-05T12:13:19,543 DEBUG [RS:0;9ea9abad0e9b:44759 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/WALs/9ea9abad0e9b,44759,1733400796278/9ea9abad0e9b%2C44759%2C1733400796278.1733400799532, exclude list is [], retry=0 2024-12-05T12:13:19,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32993,DS-4043a30f-b058-46e5-afb3-e568bf768ae5,DISK] 2024-12-05T12:13:19,551 INFO [RS:0;9ea9abad0e9b:44759 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/WALs/9ea9abad0e9b,44759,1733400796278/9ea9abad0e9b%2C44759%2C1733400796278.1733400799532 2024-12-05T12:13:19,551 DEBUG [RS:0;9ea9abad0e9b:44759 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39773:39773)] 2024-12-05T12:13:19,611 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T12:13:19,612 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e 2024-12-05T12:13:19,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741833_1009 (size=32) 2024-12-05T12:13:20,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:13:20,035 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T12:13:20,038 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T12:13:20,038 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:20,039 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:13:20,039 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T12:13:20,041 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T12:13:20,042 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:20,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:13:20,043 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T12:13:20,045 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T12:13:20,045 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:20,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:13:20,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T12:13:20,049 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T12:13:20,049 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:20,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:13:20,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T12:13:20,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740 2024-12-05T12:13:20,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740 2024-12-05T12:13:20,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T12:13:20,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T12:13:20,056 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:13:20,058 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T12:13:20,062 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:13:20,063 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74865145, jitterRate=0.1155775934457779}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:13:20,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733400800031Initializing all the Stores at 1733400800034 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400800034Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400800034Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400800035 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400800035Cleaning up temporary data from old regions at 1733400800055 (+20 ms)Region opened successfully at 1733400800064 (+9 ms) 2024-12-05T12:13:20,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T12:13:20,065 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T12:13:20,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T12:13:20,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T12:13:20,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T12:13:20,066 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T12:13:20,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733400800064Disabling compacts and flushes for region at 1733400800065 (+1 ms)Disabling writes for close at 1733400800065Writing region close event to WAL at 1733400800066 (+1 ms)Closed at 1733400800066 2024-12-05T12:13:20,069 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T12:13:20,070 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T12:13:20,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T12:13:20,083 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T12:13:20,085 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T12:13:20,243 DEBUG [9ea9abad0e9b:34103 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T12:13:20,256 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:20,264 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ea9abad0e9b,44759,1733400796278, state=OPENING 2024-12-05T12:13:20,300 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T12:13:20,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:20,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:13:20,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:13:20,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:13:20,313 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T12:13:20,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:13:20,489 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T12:13:20,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35491, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T12:13:20,506 INFO [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T12:13:20,507 INFO [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:13:20,508 INFO [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T12:13:20,512 INFO [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ea9abad0e9b%2C44759%2C1733400796278.meta, suffix=.meta, logDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/WALs/9ea9abad0e9b,44759,1733400796278, archiveDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/oldWALs, maxLogs=32 2024-12-05T12:13:20,525 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/WALs/9ea9abad0e9b,44759,1733400796278/9ea9abad0e9b%2C44759%2C1733400796278.meta.1733400800513.meta, exclude list is [], retry=0 2024-12-05T12:13:20,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32993,DS-4043a30f-b058-46e5-afb3-e568bf768ae5,DISK] 2024-12-05T12:13:20,532 INFO [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/WALs/9ea9abad0e9b,44759,1733400796278/9ea9abad0e9b%2C44759%2C1733400796278.meta.1733400800513.meta 2024-12-05T12:13:20,533 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39773:39773)] 2024-12-05T12:13:20,533 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:13:20,535 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T12:13:20,537 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T12:13:20,543 INFO [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T12:13:20,546 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T12:13:20,547 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:13:20,547 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T12:13:20,547 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T12:13:20,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T12:13:20,552 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T12:13:20,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:20,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:13:20,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T12:13:20,555 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T12:13:20,555 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:20,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:13:20,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T12:13:20,558 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T12:13:20,558 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:20,559 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:13:20,559 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T12:13:20,560 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T12:13:20,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:20,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:13:20,561 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T12:13:20,563 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740 2024-12-05T12:13:20,566 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740 2024-12-05T12:13:20,569 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T12:13:20,569 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T12:13:20,570 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:13:20,573 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T12:13:20,574 INFO [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60884966, jitterRate=-0.09274330735206604}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:13:20,575 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T12:13:20,576 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733400800548Writing region info on filesystem at 1733400800548Initializing all the Stores at 1733400800550 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400800550Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400800550Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400800550Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400800550Cleaning up temporary data from old regions at 1733400800569 (+19 ms)Running coprocessor post-open hooks at 1733400800575 (+6 ms)Region opened successfully at 1733400800576 (+1 ms) 2024-12-05T12:13:20,583 INFO [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733400800481 2024-12-05T12:13:20,593 DEBUG [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T12:13:20,594 INFO [RS_OPEN_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T12:13:20,595 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:20,598 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ea9abad0e9b,44759,1733400796278, state=OPEN 2024-12-05T12:13:20,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:13:20,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:13:20,652 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:13:20,652 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:13:20,653 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:20,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T12:13:20,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9ea9abad0e9b,44759,1733400796278 in 339 msec 2024-12-05T12:13:20,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T12:13:20,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 589 msec 2024-12-05T12:13:20,670 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T12:13:20,670 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T12:13:20,691 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:20,692 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:20,710 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:20,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59201, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:20,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6400 sec 2024-12-05T12:13:20,733 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733400800733, completionTime=-1 2024-12-05T12:13:20,736 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T12:13:20,736 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T12:13:20,761 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-05T12:13:20,761 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733400860761 2024-12-05T12:13:20,762 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733400920762 2024-12-05T12:13:20,762 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-12-05T12:13:20,764 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ea9abad0e9b,34103,1733400795471-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:20,764 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ea9abad0e9b,34103,1733400795471-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:20,764 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ea9abad0e9b,34103,1733400795471-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:20,766 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9ea9abad0e9b:34103, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:20,766 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:20,767 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:20,772 DEBUG [master/9ea9abad0e9b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T12:13:20,790 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.338sec 2024-12-05T12:13:20,791 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T12:13:20,793 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T12:13:20,793 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T12:13:20,794 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T12:13:20,794 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T12:13:20,795 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ea9abad0e9b,34103,1733400795471-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T12:13:20,795 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ea9abad0e9b,34103,1733400795471-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T12:13:20,804 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T12:13:20,805 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T12:13:20,805 INFO [master/9ea9abad0e9b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ea9abad0e9b,34103,1733400795471-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:13:20,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d57a2b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:20,897 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T12:13:20,897 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T12:13:20,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:20,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:20,909 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756bc43e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:20,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:20,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7fbb0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:20,919 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:20,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:20,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:20,987 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=444, ProcessCount=11, AvailableMemoryMB=4496 2024-12-05T12:13:21,000 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T12:13:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:21,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23cc1f5f 2024-12-05T12:13:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T12:13:21,013 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T12:13:21,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T12:13:21,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:13:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T12:13:21,028 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T12:13:21,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-05T12:13:21,031 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:21,034 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T12:13:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:13:21,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741835_1011 (size=1061) 2024-12-05T12:13:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:13:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:13:21,455 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e 2024-12-05T12:13:21,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741836_1012 (size=53) 2024-12-05T12:13:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:13:21,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:13:21,872 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9c2bbad557c305bbacb16e1dfd91bc9a, disabling compactions & flushes 2024-12-05T12:13:21,872 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:21,872 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:21,872 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. after waiting 0 ms 2024-12-05T12:13:21,872 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:21,873 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:21,873 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9c2bbad557c305bbacb16e1dfd91bc9a: Waiting for close lock at 1733400801871Disabling compacts and flushes for region at 1733400801871Disabling writes for close at 1733400801872 (+1 ms)Writing region close event to WAL at 1733400801873 (+1 ms)Closed at 1733400801873 2024-12-05T12:13:21,878 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T12:13:21,883 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733400801878"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733400801878"}]},"ts":"1733400801878"} 2024-12-05T12:13:21,887 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T12:13:21,890 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T12:13:21,892 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400801890"}]},"ts":"1733400801890"} 2024-12-05T12:13:21,897 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T12:13:21,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c2bbad557c305bbacb16e1dfd91bc9a, ASSIGN}] 2024-12-05T12:13:21,902 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c2bbad557c305bbacb16e1dfd91bc9a, ASSIGN 2024-12-05T12:13:21,903 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c2bbad557c305bbacb16e1dfd91bc9a, ASSIGN; state=OFFLINE, location=9ea9abad0e9b,44759,1733400796278; forceNewPlan=false, retain=false 2024-12-05T12:13:22,057 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9c2bbad557c305bbacb16e1dfd91bc9a, regionState=OPENING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:22,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c2bbad557c305bbacb16e1dfd91bc9a, ASSIGN because future has completed 2024-12-05T12:13:22,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:13:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:13:22,234 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:22,234 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:13:22,235 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,235 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:13:22,235 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,235 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,237 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,241 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:13:22,241 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c2bbad557c305bbacb16e1dfd91bc9a columnFamilyName A 2024-12-05T12:13:22,241 DEBUG [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:22,243 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] regionserver.HStore(327): Store=9c2bbad557c305bbacb16e1dfd91bc9a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:22,243 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,245 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:13:22,245 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c2bbad557c305bbacb16e1dfd91bc9a columnFamilyName B 2024-12-05T12:13:22,245 DEBUG [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:22,246 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] regionserver.HStore(327): Store=9c2bbad557c305bbacb16e1dfd91bc9a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:22,246 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,249 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:13:22,249 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c2bbad557c305bbacb16e1dfd91bc9a columnFamilyName C 2024-12-05T12:13:22,249 DEBUG [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:22,250 INFO [StoreOpener-9c2bbad557c305bbacb16e1dfd91bc9a-1 {}] regionserver.HStore(327): Store=9c2bbad557c305bbacb16e1dfd91bc9a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:22,251 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:22,251 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,252 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,253 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,255 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,255 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,256 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:13:22,258 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,263 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:13:22,264 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9c2bbad557c305bbacb16e1dfd91bc9a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65532645, jitterRate=-0.02348749339580536}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:13:22,264 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:22,265 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9c2bbad557c305bbacb16e1dfd91bc9a: Running coprocessor pre-open hook at 1733400802235Writing region info on filesystem at 1733400802235Initializing all the Stores at 1733400802237 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400802237Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400802237Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400802237Cleaning up temporary data from old regions at 1733400802255 (+18 ms)Running coprocessor post-open hooks at 1733400802264 (+9 ms)Region opened successfully at 1733400802265 (+1 ms) 2024-12-05T12:13:22,268 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., pid=6, masterSystemTime=1733400802224 2024-12-05T12:13:22,271 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:22,272 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:22,273 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9c2bbad557c305bbacb16e1dfd91bc9a, regionState=OPEN, openSeqNum=2, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:22,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:13:22,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T12:13:22,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 in 211 msec 2024-12-05T12:13:22,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T12:13:22,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c2bbad557c305bbacb16e1dfd91bc9a, ASSIGN in 384 msec 2024-12-05T12:13:22,288 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T12:13:22,289 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400802288"}]},"ts":"1733400802288"} 2024-12-05T12:13:22,291 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T12:13:22,293 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T12:13:22,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2720 sec 2024-12-05T12:13:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:13:23,187 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:23,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1487a66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:23,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:23,198 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:23,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@455d50bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:23,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:23,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:23,201 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da125de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,203 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:23,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:23,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:23,210 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T12:13:23,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:23,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e889155 2024-12-05T12:13:23,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T12:13:23,216 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T12:13:23,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d3f2f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:23,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:23,223 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a8011b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:23,227 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1380f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ad9e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,229 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:23,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:23,230 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6d3ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:23,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:23,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:23,233 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:23,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27cf2c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6308d33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:23,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:23,237 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46fadb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:23,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230faddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a822d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:23,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:23,244 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3c5a0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:23,248 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fdb4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7813eeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:23,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:23,251 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:23,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:23,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:23,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b03727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:23,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:23,254 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347812b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f66d26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:23,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:23,258 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:23,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:23,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:23,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad2755a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:23,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:23,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:23,261 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:23,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1475e077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@225d1960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:23,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:23,264 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355546b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:23,267 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31aa940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93b4e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:23,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:23,270 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:23,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:23,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:23,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2815dd8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:23,274 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:23,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7994a976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7449cb3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:23,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:23,278 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3714a06f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:23,280 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4689148e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:23,285 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:23,285 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:23,285 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:23,286 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:23,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:23,287 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:23,288 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:23,288 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:23,288 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:23,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:23,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:23,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:23,293 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:23,295 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:23,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:23,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:23,296 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:23,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-05T12:13:23,298 DEBUG [Thread-147 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:23,299 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:23,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:13:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T12:13:23,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:13:23,302 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:13:23,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:13:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:13:23,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:13:23,309 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:23,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:13:23,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:13:23,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:13:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:23,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T12:13:23,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:23,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:23,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:23,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:23,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:23,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T12:13:23,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T12:13:23,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:23,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:23,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:23,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:23,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/a3f4192ab9f3406a87bcfa98517e7e43 is 50, key is test_row_0/A:col10/1733400803381/Put/seqid=0 2024-12-05T12:13:23,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400863489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:23,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400863500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733400803462, max timestamp=1733400803507Num uniques 859; 2024-12-05T12:13:23,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:13:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733400803476, max timestamp=1733400803507Num uniques 859; 2024-12-05T12:13:23,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=7872, suffix heap size=117064,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=172 2024-12-05T12:13:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400803476, max timestamp=1733400803507Num uniques 859; 2024-12-05T12:13:23,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:23,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741837_1013 (size=14341) 2024-12-05T12:13:23,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/a3f4192ab9f3406a87bcfa98517e7e43 2024-12-05T12:13:23,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1733400863610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T12:13:23,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400863616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400863617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400863633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400863644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T12:13:23,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:23,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:23,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:23,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:23,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:23,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/5c793a368fe54b04996a36958d3d0b86 is 50, key is test_row_0/B:col10/1733400803381/Put/seqid=0 2024-12-05T12:13:23,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1733400863734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741838_1014 (size=12001) 2024-12-05T12:13:23,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400863743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400863744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400863843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T12:13:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:23,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:23,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400863864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T12:13:23,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1733400863944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400863964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400863964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T12:13:24,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:24,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:24,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:24,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:24,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:24,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/5c793a368fe54b04996a36958d3d0b86 2024-12-05T12:13:24,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400864154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T12:13:24,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:24,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:24,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:24,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:24,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:24,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400864174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/226669c6838b4353b8a05e7049fda68f is 50, key is test_row_0/C:col10/1733400803381/Put/seqid=0 2024-12-05T12:13:24,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741839_1015 (size=12001) 2024-12-05T12:13:24,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/226669c6838b4353b8a05e7049fda68f 2024-12-05T12:13:24,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/a3f4192ab9f3406a87bcfa98517e7e43 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a3f4192ab9f3406a87bcfa98517e7e43 2024-12-05T12:13:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1733400864264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a3f4192ab9f3406a87bcfa98517e7e43, entries=200, sequenceid=14, filesize=14.0 K 2024-12-05T12:13:24,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/5c793a368fe54b04996a36958d3d0b86 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/5c793a368fe54b04996a36958d3d0b86 2024-12-05T12:13:24,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/5c793a368fe54b04996a36958d3d0b86, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T12:13:24,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/226669c6838b4353b8a05e7049fda68f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/226669c6838b4353b8a05e7049fda68f 2024-12-05T12:13:24,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/226669c6838b4353b8a05e7049fda68f, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T12:13:24,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=112.40 KB/115098 for 9c2bbad557c305bbacb16e1dfd91bc9a in 926ms, sequenceid=14, compaction requested=false 2024-12-05T12:13:24,318 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T12:13:24,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:24,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T12:13:24,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:24,325 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=112.40 KB heapSize=316.77 KB 2024-12-05T12:13:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:24,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:24,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:24,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/d82f368c0d03411791aeba52aa27e72f is 50, key is test_row_0/A:col10/1733400803611/Put/seqid=0 2024-12-05T12:13:24,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741840_1016 (size=14341) 2024-12-05T12:13:24,396 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/d82f368c0d03411791aeba52aa27e72f 2024-12-05T12:13:24,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/6bb0734edc0040209a220618322f7388 is 50, key is test_row_0/B:col10/1733400803611/Put/seqid=0 2024-12-05T12:13:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T12:13:24,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741841_1017 (size=12001) 2024-12-05T12:13:24,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400864461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400864479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400864595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400864615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400864674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400864694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1733400864774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400864824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400864834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:24,857 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/6bb0734edc0040209a220618322f7388 2024-12-05T12:13:24,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ee0e946ab5eb47beb689dd5c05db9638 is 50, key is test_row_0/C:col10/1733400803611/Put/seqid=0 2024-12-05T12:13:24,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741842_1018 (size=12001) 2024-12-05T12:13:24,939 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ee0e946ab5eb47beb689dd5c05db9638 2024-12-05T12:13:24,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/d82f368c0d03411791aeba52aa27e72f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/d82f368c0d03411791aeba52aa27e72f 2024-12-05T12:13:24,970 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/d82f368c0d03411791aeba52aa27e72f, entries=200, sequenceid=48, filesize=14.0 K 2024-12-05T12:13:24,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/6bb0734edc0040209a220618322f7388 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/6bb0734edc0040209a220618322f7388 2024-12-05T12:13:24,998 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/6bb0734edc0040209a220618322f7388, entries=150, sequenceid=48, filesize=11.7 K 2024-12-05T12:13:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ee0e946ab5eb47beb689dd5c05db9638 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ee0e946ab5eb47beb689dd5c05db9638 2024-12-05T12:13:25,021 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ee0e946ab5eb47beb689dd5c05db9638, entries=150, sequenceid=48, filesize=11.7 K 2024-12-05T12:13:25,023 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~119.11 KB/121968, heapSize ~336.65 KB/344728, currentSize=60.38 KB/61830 for 9c2bbad557c305bbacb16e1dfd91bc9a in 699ms, sequenceid=48, compaction requested=false 2024-12-05T12:13:25,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:25,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:25,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-05T12:13:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-05T12:13:25,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T12:13:25,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7130 sec 2024-12-05T12:13:25,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7520 sec 2024-12-05T12:13:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:25,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T12:13:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:25,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/3ee859e669f84e7c9b78c9732405300e is 50, key is test_row_0/A:col10/1733400805157/Put/seqid=0 2024-12-05T12:13:25,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741843_1019 (size=16681) 2024-12-05T12:13:25,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/3ee859e669f84e7c9b78c9732405300e 2024-12-05T12:13:25,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/b2ae764790224f1f8f758b79487b455c is 50, key is test_row_0/B:col10/1733400805157/Put/seqid=0 2024-12-05T12:13:25,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741844_1020 (size=12001) 2024-12-05T12:13:25,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/b2ae764790224f1f8f758b79487b455c 2024-12-05T12:13:25,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400865280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:25,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:25,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400865283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:25,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/adfaf98dc484484081557d031a98e2ba is 50, key is test_row_0/C:col10/1733400805157/Put/seqid=0 2024-12-05T12:13:25,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741845_1021 (size=12001) 2024-12-05T12:13:25,348 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T12:13:25,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:25,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400865403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400865404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T12:13:25,453 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:25,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-05T12:13:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T12:13:25,458 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:25,462 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:25,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T12:13:25,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T12:13:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:25,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:25,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:25,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:25,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:25,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400865613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:25,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400865613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:25,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400865703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:25,711 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:25,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:25,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400865723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:25,727 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:25,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/adfaf98dc484484081557d031a98e2ba 2024-12-05T12:13:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T12:13:25,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/3ee859e669f84e7c9b78c9732405300e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/3ee859e669f84e7c9b78c9732405300e 2024-12-05T12:13:25,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T12:13:25,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:25,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:25,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:25,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:25,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:25,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/3ee859e669f84e7c9b78c9732405300e, entries=250, sequenceid=62, filesize=16.3 K 2024-12-05T12:13:25,793 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-05T12:13:25,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/b2ae764790224f1f8f758b79487b455c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/b2ae764790224f1f8f758b79487b455c 2024-12-05T12:13:25,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/b2ae764790224f1f8f758b79487b455c, entries=150, sequenceid=62, filesize=11.7 K 2024-12-05T12:13:25,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/adfaf98dc484484081557d031a98e2ba as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/adfaf98dc484484081557d031a98e2ba 2024-12-05T12:13:25,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1733400865823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:25,832 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:25,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/adfaf98dc484484081557d031a98e2ba, entries=150, sequenceid=62, filesize=11.7 K 2024-12-05T12:13:25,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9c2bbad557c305bbacb16e1dfd91bc9a in 689ms, sequenceid=62, compaction requested=true 2024-12-05T12:13:25,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:25,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:25,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:25,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:25,855 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:25,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:25,855 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:25,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:25,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:25,859 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:25,861 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:25,862 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:25,864 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/5c793a368fe54b04996a36958d3d0b86, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/6bb0734edc0040209a220618322f7388, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/b2ae764790224f1f8f758b79487b455c] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=35.2 K 2024-12-05T12:13:25,866 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c793a368fe54b04996a36958d3d0b86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733400803327 2024-12-05T12:13:25,868 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb0734edc0040209a220618322f7388, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733400803603 2024-12-05T12:13:25,870 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ae764790224f1f8f758b79487b455c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733400804413 2024-12-05T12:13:25,872 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:25,873 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:25,873 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:25,873 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a3f4192ab9f3406a87bcfa98517e7e43, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/d82f368c0d03411791aeba52aa27e72f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/3ee859e669f84e7c9b78c9732405300e] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=44.3 K 2024-12-05T12:13:25,874 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3f4192ab9f3406a87bcfa98517e7e43, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733400803327 2024-12-05T12:13:25,875 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting d82f368c0d03411791aeba52aa27e72f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733400803603 2024-12-05T12:13:25,876 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ee859e669f84e7c9b78c9732405300e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733400804413 2024-12-05T12:13:25,924 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#10 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:25,924 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#9 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:25,926 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/f06ac642462e43ad94013bb514bb03f1 is 50, key is test_row_0/B:col10/1733400805157/Put/seqid=0 2024-12-05T12:13:25,929 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/70775de9ac854fe481ee4d0693cc92fd is 50, key is test_row_0/A:col10/1733400805157/Put/seqid=0 2024-12-05T12:13:25,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T12:13:25,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:25,937 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T12:13:25,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:25,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:25,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:25,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:25,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:25,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:25,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:25,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741846_1022 (size=12104) 2024-12-05T12:13:25,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/457263c9db4544598136a46520b8117e is 50, key is test_row_0/A:col10/1733400805935/Put/seqid=0 2024-12-05T12:13:25,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741847_1023 (size=12104) 2024-12-05T12:13:26,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400866000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:26,009 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/70775de9ac854fe481ee4d0693cc92fd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/70775de9ac854fe481ee4d0693cc92fd 2024-12-05T12:13:26,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400866008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:26,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741848_1024 (size=14341) 2024-12-05T12:13:26,030 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/457263c9db4544598136a46520b8117e 2024-12-05T12:13:26,046 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into 70775de9ac854fe481ee4d0693cc92fd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:26,046 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:26,050 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=13, startTime=1733400805851; duration=0sec 2024-12-05T12:13:26,051 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:26,051 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:26,051 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:26,054 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:26,054 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:26,054 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:26,055 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/226669c6838b4353b8a05e7049fda68f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ee0e946ab5eb47beb689dd5c05db9638, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/adfaf98dc484484081557d031a98e2ba] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=35.2 K 2024-12-05T12:13:26,057 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 226669c6838b4353b8a05e7049fda68f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733400803327 2024-12-05T12:13:26,058 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee0e946ab5eb47beb689dd5c05db9638, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733400803603 2024-12-05T12:13:26,060 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T12:13:26,061 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting adfaf98dc484484081557d031a98e2ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733400804413 2024-12-05T12:13:26,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/4072a8254d9a46d6bcb4dde4c8e3b58d is 50, key is test_row_0/B:col10/1733400805935/Put/seqid=0 2024-12-05T12:13:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T12:13:26,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741849_1025 (size=12001) 2024-12-05T12:13:26,106 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/4072a8254d9a46d6bcb4dde4c8e3b58d 2024-12-05T12:13:26,109 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:26,111 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/b3ca07680b3d40458b48a5760efbcd2a is 50, key is test_row_0/C:col10/1733400805157/Put/seqid=0 2024-12-05T12:13:26,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400866115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:26,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400866123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:26,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T12:13:26,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T12:13:26,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T12:13:26,156 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T12:13:26,157 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T12:13:26,157 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T12:13:26,159 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T12:13:26,159 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T12:13:26,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741850_1026 (size=12104) 2024-12-05T12:13:26,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/b34454f575714ea09f97feec74515e20 is 50, key is test_row_0/C:col10/1733400805935/Put/seqid=0 2024-12-05T12:13:26,184 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/b3ca07680b3d40458b48a5760efbcd2a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b3ca07680b3d40458b48a5760efbcd2a 2024-12-05T12:13:26,200 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into b3ca07680b3d40458b48a5760efbcd2a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:26,200 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:26,200 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=13, startTime=1733400805855; duration=0sec 2024-12-05T12:13:26,200 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:26,200 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:26,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741851_1027 (size=12001) 2024-12-05T12:13:26,210 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/b34454f575714ea09f97feec74515e20 2024-12-05T12:13:26,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/457263c9db4544598136a46520b8117e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/457263c9db4544598136a46520b8117e 2024-12-05T12:13:26,239 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/457263c9db4544598136a46520b8117e, entries=200, sequenceid=81, filesize=14.0 K 2024-12-05T12:13:26,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/4072a8254d9a46d6bcb4dde4c8e3b58d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4072a8254d9a46d6bcb4dde4c8e3b58d 2024-12-05T12:13:26,266 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4072a8254d9a46d6bcb4dde4c8e3b58d, entries=150, sequenceid=81, filesize=11.7 K 2024-12-05T12:13:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/b34454f575714ea09f97feec74515e20 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b34454f575714ea09f97feec74515e20 2024-12-05T12:13:26,283 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b34454f575714ea09f97feec74515e20, entries=150, sequenceid=81, filesize=11.7 K 2024-12-05T12:13:26,286 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9c2bbad557c305bbacb16e1dfd91bc9a in 349ms, sequenceid=81, compaction requested=false 2024-12-05T12:13:26,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:26,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:26,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-05T12:13:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-05T12:13:26,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T12:13:26,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 827 msec 2024-12-05T12:13:26,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 842 msec 2024-12-05T12:13:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:26,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T12:13:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:26,391 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/f06ac642462e43ad94013bb514bb03f1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f06ac642462e43ad94013bb514bb03f1 2024-12-05T12:13:26,409 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into f06ac642462e43ad94013bb514bb03f1(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:26,410 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:26,410 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=13, startTime=1733400805855; duration=0sec 2024-12-05T12:13:26,410 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:26,410 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:26,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/103b3fac3b224bb4a12ffa25ffeb56df is 50, key is test_row_0/A:col10/1733400806338/Put/seqid=0 2024-12-05T12:13:26,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741852_1028 (size=12001) 2024-12-05T12:13:26,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400866480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:26,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400866483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:26,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T12:13:26,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:26,594 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:26,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400866594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400866594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-05T12:13:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T12:13:26,602 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:26,604 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:26,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T12:13:26,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T12:13:26,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:26,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:26,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:26,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400866803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:26,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400866803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:26,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/103b3fac3b224bb4a12ffa25ffeb56df 2024-12-05T12:13:26,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/2e5c5863806643bcbef4decec42d9fa7 is 50, key is test_row_0/B:col10/1733400806338/Put/seqid=0 2024-12-05T12:13:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T12:13:26,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741853_1029 (size=12001) 2024-12-05T12:13:26,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T12:13:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:26,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:26,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:26,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:26,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/2e5c5863806643bcbef4decec42d9fa7 2024-12-05T12:13:26,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ba50508fcf4242f2bd0a10bfc4aa185d is 50, key is test_row_0/C:col10/1733400806338/Put/seqid=0 2024-12-05T12:13:26,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741854_1030 (size=12001) 2024-12-05T12:13:26,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ba50508fcf4242f2bd0a10bfc4aa185d 2024-12-05T12:13:26,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/103b3fac3b224bb4a12ffa25ffeb56df as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/103b3fac3b224bb4a12ffa25ffeb56df 2024-12-05T12:13:27,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/103b3fac3b224bb4a12ffa25ffeb56df, entries=150, sequenceid=98, filesize=11.7 K 2024-12-05T12:13:27,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/2e5c5863806643bcbef4decec42d9fa7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/2e5c5863806643bcbef4decec42d9fa7 2024-12-05T12:13:27,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/2e5c5863806643bcbef4decec42d9fa7, entries=150, sequenceid=98, filesize=11.7 K 2024-12-05T12:13:27,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ba50508fcf4242f2bd0a10bfc4aa185d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ba50508fcf4242f2bd0a10bfc4aa185d 2024-12-05T12:13:27,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ba50508fcf4242f2bd0a10bfc4aa185d, entries=150, sequenceid=98, filesize=11.7 K 2024-12-05T12:13:27,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9c2bbad557c305bbacb16e1dfd91bc9a in 658ms, sequenceid=98, compaction requested=true 2024-12-05T12:13:27,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:27,041 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:27,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:27,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:27,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:27,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:27,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:27,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T12:13:27,042 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:27,044 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:27,044 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:27,044 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:27,044 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:27,044 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:27,044 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:27,044 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/70775de9ac854fe481ee4d0693cc92fd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/457263c9db4544598136a46520b8117e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/103b3fac3b224bb4a12ffa25ffeb56df] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=37.5 K 2024-12-05T12:13:27,044 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f06ac642462e43ad94013bb514bb03f1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4072a8254d9a46d6bcb4dde4c8e3b58d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/2e5c5863806643bcbef4decec42d9fa7] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=35.3 K 2024-12-05T12:13:27,045 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting f06ac642462e43ad94013bb514bb03f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733400804413 2024-12-05T12:13:27,045 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 70775de9ac854fe481ee4d0693cc92fd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733400804413 2024-12-05T12:13:27,046 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4072a8254d9a46d6bcb4dde4c8e3b58d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733400805215 2024-12-05T12:13:27,046 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 457263c9db4544598136a46520b8117e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733400805215 2024-12-05T12:13:27,046 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e5c5863806643bcbef4decec42d9fa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733400805996 2024-12-05T12:13:27,047 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 103b3fac3b224bb4a12ffa25ffeb56df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733400805996 2024-12-05T12:13:27,065 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:27,065 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:27,066 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/ea043dfb4068485b976605709f4eb838 is 50, key is test_row_0/A:col10/1733400806338/Put/seqid=0 2024-12-05T12:13:27,066 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/67722cd9e398491ca3ad73735864dc66 is 50, key is test_row_0/B:col10/1733400806338/Put/seqid=0 2024-12-05T12:13:27,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T12:13:27,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:27,074 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T12:13:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:27,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741856_1032 (size=12207) 2024-12-05T12:13:27,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741855_1031 (size=12207) 2024-12-05T12:13:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/86e6d71c83cf4f25bb7ea2e5f0b9e022 is 50, key is test_row_0/A:col10/1733400806461/Put/seqid=0 2024-12-05T12:13:27,098 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/ea043dfb4068485b976605709f4eb838 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/ea043dfb4068485b976605709f4eb838 2024-12-05T12:13:27,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741857_1033 (size=12001) 2024-12-05T12:13:27,115 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into ea043dfb4068485b976605709f4eb838(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:27,115 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:27,115 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=13, startTime=1733400807041; duration=0sec 2024-12-05T12:13:27,116 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:27,116 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:27,116 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:27,119 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:27,119 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:27,120 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:27,120 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b3ca07680b3d40458b48a5760efbcd2a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b34454f575714ea09f97feec74515e20, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ba50508fcf4242f2bd0a10bfc4aa185d] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=35.3 K 2024-12-05T12:13:27,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:27,122 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b3ca07680b3d40458b48a5760efbcd2a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733400804413 2024-12-05T12:13:27,124 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b34454f575714ea09f97feec74515e20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733400805215 2024-12-05T12:13:27,126 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting ba50508fcf4242f2bd0a10bfc4aa185d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733400805996 2024-12-05T12:13:27,149 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:27,150 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/24bcaa9df6b141a09f2792a59e66d3f9 is 50, key is test_row_0/C:col10/1733400806338/Put/seqid=0 2024-12-05T12:13:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400867164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741858_1034 (size=12207) 2024-12-05T12:13:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400867168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,182 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/24bcaa9df6b141a09f2792a59e66d3f9 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/24bcaa9df6b141a09f2792a59e66d3f9 2024-12-05T12:13:27,201 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into 24bcaa9df6b141a09f2792a59e66d3f9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:27,201 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:27,201 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=13, startTime=1733400807042; duration=0sec 2024-12-05T12:13:27,202 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:27,202 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T12:13:27,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400867273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400867282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400867484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,492 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/67722cd9e398491ca3ad73735864dc66 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/67722cd9e398491ca3ad73735864dc66 2024-12-05T12:13:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400867493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,507 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into 67722cd9e398491ca3ad73735864dc66(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:27,507 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:27,508 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=13, startTime=1733400807041; duration=0sec 2024-12-05T12:13:27,508 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:27,508 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:27,510 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/86e6d71c83cf4f25bb7ea2e5f0b9e022 2024-12-05T12:13:27,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/efce2637dbf54edaaf03274f707d071e is 50, key is test_row_0/B:col10/1733400806461/Put/seqid=0 2024-12-05T12:13:27,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741859_1035 (size=12001) 2024-12-05T12:13:27,558 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/efce2637dbf54edaaf03274f707d071e 2024-12-05T12:13:27,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/c1fd03abac004c2193fc60efb1c30f64 is 50, key is test_row_0/C:col10/1733400806461/Put/seqid=0 2024-12-05T12:13:27,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741860_1036 (size=12001) 2024-12-05T12:13:27,593 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/c1fd03abac004c2193fc60efb1c30f64 2024-12-05T12:13:27,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/86e6d71c83cf4f25bb7ea2e5f0b9e022 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/86e6d71c83cf4f25bb7ea2e5f0b9e022 2024-12-05T12:13:27,618 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/86e6d71c83cf4f25bb7ea2e5f0b9e022, entries=150, sequenceid=116, filesize=11.7 K 2024-12-05T12:13:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/efce2637dbf54edaaf03274f707d071e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/efce2637dbf54edaaf03274f707d071e 2024-12-05T12:13:27,637 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/efce2637dbf54edaaf03274f707d071e, entries=150, sequenceid=116, filesize=11.7 K 2024-12-05T12:13:27,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/c1fd03abac004c2193fc60efb1c30f64 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c1fd03abac004c2193fc60efb1c30f64 2024-12-05T12:13:27,660 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c1fd03abac004c2193fc60efb1c30f64, entries=150, sequenceid=116, filesize=11.7 K 2024-12-05T12:13:27,662 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9c2bbad557c305bbacb16e1dfd91bc9a in 588ms, sequenceid=116, compaction requested=false 2024-12-05T12:13:27,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:27,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:27,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-05T12:13:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-05T12:13:27,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-05T12:13:27,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0600 sec 2024-12-05T12:13:27,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0750 sec 2024-12-05T12:13:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:27,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T12:13:27,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:27,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:27,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:27,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:27,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:27,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T12:13:27,733 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:27,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:27,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/5989c33d27ad4808bfa509c2951ee0bd is 50, key is test_row_0/A:col10/1733400807148/Put/seqid=0 2024-12-05T12:13:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-05T12:13:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T12:13:27,745 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:27,746 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:27,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:27,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741861_1037 (size=14491) 2024-12-05T12:13:27,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/5989c33d27ad4808bfa509c2951ee0bd 2024-12-05T12:13:27,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/c108df6bb4e74315a6ca70ba99c0da7c is 50, key is test_row_0/B:col10/1733400807148/Put/seqid=0 2024-12-05T12:13:27,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741862_1038 (size=12101) 2024-12-05T12:13:27,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400867803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400867804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400867804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400867806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T12:13:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1733400867853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,855 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T12:13:27,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:27,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:27,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:27,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:27,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:27,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400867913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400867914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:28,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T12:13:28,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:28,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T12:13:28,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400868123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400868124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:28,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/c108df6bb4e74315a6ca70ba99c0da7c 2024-12-05T12:13:28,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T12:13:28,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:28,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/c63b46f53d474e3eb5af43371b9bea0b is 50, key is test_row_0/C:col10/1733400807148/Put/seqid=0 2024-12-05T12:13:28,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741863_1039 (size=12101) 2024-12-05T12:13:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400868313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:28,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400868316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:28,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T12:13:28,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:28,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T12:13:28,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400868434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:28,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400868434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:28,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T12:13:28,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:28,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/c63b46f53d474e3eb5af43371b9bea0b 2024-12-05T12:13:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/5989c33d27ad4808bfa509c2951ee0bd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/5989c33d27ad4808bfa509c2951ee0bd 2024-12-05T12:13:28,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/5989c33d27ad4808bfa509c2951ee0bd, entries=200, sequenceid=135, filesize=14.2 K 2024-12-05T12:13:28,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/c108df6bb4e74315a6ca70ba99c0da7c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c108df6bb4e74315a6ca70ba99c0da7c 2024-12-05T12:13:28,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c108df6bb4e74315a6ca70ba99c0da7c, entries=150, sequenceid=135, filesize=11.8 K 2024-12-05T12:13:28,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/c63b46f53d474e3eb5af43371b9bea0b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c63b46f53d474e3eb5af43371b9bea0b 2024-12-05T12:13:28,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T12:13:28,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:28,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:28,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c63b46f53d474e3eb5af43371b9bea0b, entries=150, sequenceid=135, filesize=11.8 K 2024-12-05T12:13:28,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9c2bbad557c305bbacb16e1dfd91bc9a in 956ms, sequenceid=135, compaction requested=true 2024-12-05T12:13:28,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:28,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:28,684 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:28,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:28,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:28,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:28,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:28,684 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:28,686 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:28,686 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:28,687 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,687 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:28,687 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:28,687 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,688 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/67722cd9e398491ca3ad73735864dc66, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/efce2637dbf54edaaf03274f707d071e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c108df6bb4e74315a6ca70ba99c0da7c] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=35.5 K 2024-12-05T12:13:28,688 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/ea043dfb4068485b976605709f4eb838, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/86e6d71c83cf4f25bb7ea2e5f0b9e022, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/5989c33d27ad4808bfa509c2951ee0bd] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=37.8 K 2024-12-05T12:13:28,689 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 67722cd9e398491ca3ad73735864dc66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733400805996 2024-12-05T12:13:28,689 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea043dfb4068485b976605709f4eb838, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733400805996 2024-12-05T12:13:28,690 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting efce2637dbf54edaaf03274f707d071e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733400806461 2024-12-05T12:13:28,691 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting c108df6bb4e74315a6ca70ba99c0da7c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733400807148 2024-12-05T12:13:28,691 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86e6d71c83cf4f25bb7ea2e5f0b9e022, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733400806461 2024-12-05T12:13:28,692 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5989c33d27ad4808bfa509c2951ee0bd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733400807148 2024-12-05T12:13:28,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:28,721 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:28,723 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/f9aa4514356c475fa9d75bc0d6f5876f is 50, key is test_row_0/A:col10/1733400807148/Put/seqid=0 2024-12-05T12:13:28,729 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:28,730 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7b65b896f2ae44c9b4036be750103980 is 50, key is test_row_0/B:col10/1733400807148/Put/seqid=0 2024-12-05T12:13:28,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741864_1040 (size=12409) 2024-12-05T12:13:28,754 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/f9aa4514356c475fa9d75bc0d6f5876f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/f9aa4514356c475fa9d75bc0d6f5876f 2024-12-05T12:13:28,765 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into f9aa4514356c475fa9d75bc0d6f5876f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:28,765 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:28,765 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=13, startTime=1733400808684; duration=0sec 2024-12-05T12:13:28,766 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:28,766 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:28,767 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:28,768 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:28,768 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:28,769 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,769 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/24bcaa9df6b141a09f2792a59e66d3f9, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c1fd03abac004c2193fc60efb1c30f64, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c63b46f53d474e3eb5af43371b9bea0b] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=35.5 K 2024-12-05T12:13:28,770 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24bcaa9df6b141a09f2792a59e66d3f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733400805996 2024-12-05T12:13:28,770 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1fd03abac004c2193fc60efb1c30f64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733400806461 2024-12-05T12:13:28,771 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting c63b46f53d474e3eb5af43371b9bea0b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733400807148 2024-12-05T12:13:28,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741865_1041 (size=12409) 2024-12-05T12:13:28,790 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7b65b896f2ae44c9b4036be750103980 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7b65b896f2ae44c9b4036be750103980 2024-12-05T12:13:28,801 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:28,802 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/46ead7304d774ca190a420a339bfd89a is 50, key is test_row_0/C:col10/1733400807148/Put/seqid=0 2024-12-05T12:13:28,805 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into 7b65b896f2ae44c9b4036be750103980(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:28,805 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:28,805 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=13, startTime=1733400808684; duration=0sec 2024-12-05T12:13:28,805 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:28,805 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:28,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741866_1042 (size=12409) 2024-12-05T12:13:28,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T12:13:28,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:28,834 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T12:13:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:28,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:28,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:28,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/b1f771f712b4436da3981fd356d04fa7 is 50, key is test_row_0/A:col10/1733400807796/Put/seqid=0 2024-12-05T12:13:28,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741867_1043 (size=12151) 2024-12-05T12:13:28,865 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/b1f771f712b4436da3981fd356d04fa7 2024-12-05T12:13:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T12:13:28,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7663c02549e54c0696ba4ddff2d64fe4 is 50, key is test_row_0/B:col10/1733400807796/Put/seqid=0 2024-12-05T12:13:28,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741868_1044 (size=12151) 2024-12-05T12:13:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:28,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:28,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400868985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400868987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400869092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400869103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,230 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/46ead7304d774ca190a420a339bfd89a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/46ead7304d774ca190a420a339bfd89a 2024-12-05T12:13:29,242 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into 46ead7304d774ca190a420a339bfd89a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:29,242 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:29,243 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=13, startTime=1733400808684; duration=0sec 2024-12-05T12:13:29,243 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:29,243 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:29,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7663c02549e54c0696ba4ddff2d64fe4 2024-12-05T12:13:29,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400869303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400869314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/5bc107a911fd4b418bc891af9200a434 is 50, key is test_row_0/C:col10/1733400807796/Put/seqid=0 2024-12-05T12:13:29,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400869323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,325 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400869333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,336 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741869_1045 (size=12151) 2024-12-05T12:13:29,348 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/5bc107a911fd4b418bc891af9200a434 2024-12-05T12:13:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/b1f771f712b4436da3981fd356d04fa7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b1f771f712b4436da3981fd356d04fa7 2024-12-05T12:13:29,378 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b1f771f712b4436da3981fd356d04fa7, entries=150, sequenceid=153, filesize=11.9 K 2024-12-05T12:13:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7663c02549e54c0696ba4ddff2d64fe4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7663c02549e54c0696ba4ddff2d64fe4 2024-12-05T12:13:29,400 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7663c02549e54c0696ba4ddff2d64fe4, entries=150, sequenceid=153, filesize=11.9 K 2024-12-05T12:13:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/5bc107a911fd4b418bc891af9200a434 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/5bc107a911fd4b418bc891af9200a434 2024-12-05T12:13:29,413 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/5bc107a911fd4b418bc891af9200a434, entries=150, sequenceid=153, filesize=11.9 K 2024-12-05T12:13:29,415 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9c2bbad557c305bbacb16e1dfd91bc9a in 581ms, sequenceid=153, compaction requested=false 2024-12-05T12:13:29,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:29,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:29,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-05T12:13:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-05T12:13:29,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T12:13:29,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6710 sec 2024-12-05T12:13:29,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6930 sec 2024-12-05T12:13:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:29,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T12:13:29,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:29,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:29,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:29,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:29,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:29,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:29,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/eec0c0b0e08a4f4fa4f077c09ebd9394 is 50, key is test_row_0/A:col10/1733400809624/Put/seqid=0 2024-12-05T12:13:29,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400869656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400869658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741870_1046 (size=14541) 2024-12-05T12:13:29,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/eec0c0b0e08a4f4fa4f077c09ebd9394 2024-12-05T12:13:29,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/bad0ce9f440141a3937d707585dc7c4c is 50, key is test_row_0/B:col10/1733400809624/Put/seqid=0 2024-12-05T12:13:29,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741871_1047 (size=12151) 2024-12-05T12:13:29,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/bad0ce9f440141a3937d707585dc7c4c 2024-12-05T12:13:29,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/3c04993f0e814b70bc5701aaa7973f6b is 50, key is test_row_0/C:col10/1733400809624/Put/seqid=0 2024-12-05T12:13:29,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741872_1048 (size=12151) 2024-12-05T12:13:29,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400869773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400869773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T12:13:29,893 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:29,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-05T12:13:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T12:13:29,905 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:29,907 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:29,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:29,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400869983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:29,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400869983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:29,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T12:13:30,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T12:13:30,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:30,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:30,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:30,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/3c04993f0e814b70bc5701aaa7973f6b 2024-12-05T12:13:30,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/eec0c0b0e08a4f4fa4f077c09ebd9394 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/eec0c0b0e08a4f4fa4f077c09ebd9394 2024-12-05T12:13:30,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/eec0c0b0e08a4f4fa4f077c09ebd9394, entries=200, sequenceid=171, filesize=14.2 K 2024-12-05T12:13:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/bad0ce9f440141a3937d707585dc7c4c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/bad0ce9f440141a3937d707585dc7c4c 2024-12-05T12:13:30,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/bad0ce9f440141a3937d707585dc7c4c, entries=150, sequenceid=171, filesize=11.9 K 2024-12-05T12:13:30,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/3c04993f0e814b70bc5701aaa7973f6b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/3c04993f0e814b70bc5701aaa7973f6b 2024-12-05T12:13:30,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/3c04993f0e814b70bc5701aaa7973f6b, entries=150, sequenceid=171, filesize=11.9 K 2024-12-05T12:13:30,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9c2bbad557c305bbacb16e1dfd91bc9a in 567ms, sequenceid=171, compaction requested=true 2024-12-05T12:13:30,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:30,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:30,193 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:30,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:30,193 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:30,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:30,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:30,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:30,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:30,194 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:30,195 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:30,195 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:30,195 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,195 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:30,195 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,195 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7b65b896f2ae44c9b4036be750103980, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7663c02549e54c0696ba4ddff2d64fe4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/bad0ce9f440141a3937d707585dc7c4c] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=35.9 K 2024-12-05T12:13:30,195 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/f9aa4514356c475fa9d75bc0d6f5876f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b1f771f712b4436da3981fd356d04fa7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/eec0c0b0e08a4f4fa4f077c09ebd9394] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=38.2 K 2024-12-05T12:13:30,196 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b65b896f2ae44c9b4036be750103980, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733400807148 2024-12-05T12:13:30,196 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9aa4514356c475fa9d75bc0d6f5876f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733400807148 2024-12-05T12:13:30,197 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 7663c02549e54c0696ba4ddff2d64fe4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733400807779 2024-12-05T12:13:30,197 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1f771f712b4436da3981fd356d04fa7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733400807779 2024-12-05T12:13:30,198 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting eec0c0b0e08a4f4fa4f077c09ebd9394, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733400808967 2024-12-05T12:13:30,199 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting bad0ce9f440141a3937d707585dc7c4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733400808967 2024-12-05T12:13:30,216 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:30,217 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/c0e6fc1e44424c8e9c0b604e18378101 is 50, key is test_row_0/A:col10/1733400809624/Put/seqid=0 2024-12-05T12:13:30,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T12:13:30,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,219 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T12:13:30,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:30,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:30,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:30,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:30,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:30,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T12:13:30,237 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:30,238 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/f35c72c2710548de9f1c9a81916b6b37 is 50, key is test_row_0/B:col10/1733400809624/Put/seqid=0 2024-12-05T12:13:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/80d9211931c64f5db96dab99dbac5395 is 50, key is test_row_0/A:col10/1733400809652/Put/seqid=0 2024-12-05T12:13:30,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741873_1049 (size=12561) 2024-12-05T12:13:30,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741874_1050 (size=12561) 2024-12-05T12:13:30,273 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/f35c72c2710548de9f1c9a81916b6b37 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f35c72c2710548de9f1c9a81916b6b37 2024-12-05T12:13:30,273 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/c0e6fc1e44424c8e9c0b604e18378101 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c0e6fc1e44424c8e9c0b604e18378101 2024-12-05T12:13:30,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741875_1051 (size=12151) 2024-12-05T12:13:30,287 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into f35c72c2710548de9f1c9a81916b6b37(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:30,287 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:30,287 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=13, startTime=1733400810193; duration=0sec 2024-12-05T12:13:30,288 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:30,288 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:30,288 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:30,289 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/80d9211931c64f5db96dab99dbac5395 2024-12-05T12:13:30,293 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into c0e6fc1e44424c8e9c0b604e18378101(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:30,293 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:30,293 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=13, startTime=1733400810193; duration=0sec 2024-12-05T12:13:30,293 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:30,294 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:30,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:30,299 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:30,302 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:30,302 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,303 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/46ead7304d774ca190a420a339bfd89a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/5bc107a911fd4b418bc891af9200a434, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/3c04993f0e814b70bc5701aaa7973f6b] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=35.9 K 2024-12-05T12:13:30,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/4962cf232b604fe9b39849086eebb162 is 50, key is test_row_0/B:col10/1733400809652/Put/seqid=0 2024-12-05T12:13:30,307 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 46ead7304d774ca190a420a339bfd89a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733400807148 2024-12-05T12:13:30,309 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bc107a911fd4b418bc891af9200a434, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733400807779 2024-12-05T12:13:30,311 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c04993f0e814b70bc5701aaa7973f6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733400808967 2024-12-05T12:13:30,362 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:30,363 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/e9544160d02c4dcea8fe0091951dd71c is 50, key is test_row_0/C:col10/1733400809624/Put/seqid=0 2024-12-05T12:13:30,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741876_1052 (size=12151) 2024-12-05T12:13:30,367 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/4962cf232b604fe9b39849086eebb162 2024-12-05T12:13:30,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400870377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:30,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400870382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:30,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741877_1053 (size=12561) 2024-12-05T12:13:30,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/a3d308c15f4743b9b5646fd09877fdaa is 50, key is test_row_0/C:col10/1733400809652/Put/seqid=0 2024-12-05T12:13:30,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741878_1054 (size=12151) 2024-12-05T12:13:30,432 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/a3d308c15f4743b9b5646fd09877fdaa 2024-12-05T12:13:30,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/80d9211931c64f5db96dab99dbac5395 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/80d9211931c64f5db96dab99dbac5395 2024-12-05T12:13:30,459 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/80d9211931c64f5db96dab99dbac5395, entries=150, sequenceid=186, filesize=11.9 K 2024-12-05T12:13:30,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/4962cf232b604fe9b39849086eebb162 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4962cf232b604fe9b39849086eebb162 2024-12-05T12:13:30,475 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4962cf232b604fe9b39849086eebb162, entries=150, sequenceid=186, filesize=11.9 K 2024-12-05T12:13:30,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/a3d308c15f4743b9b5646fd09877fdaa as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/a3d308c15f4743b9b5646fd09877fdaa 2024-12-05T12:13:30,489 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/a3d308c15f4743b9b5646fd09877fdaa, entries=150, sequenceid=186, filesize=11.9 K 2024-12-05T12:13:30,491 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9c2bbad557c305bbacb16e1dfd91bc9a in 272ms, sequenceid=186, compaction requested=false 2024-12-05T12:13:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-05T12:13:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-05T12:13:30,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T12:13:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:30,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-05T12:13:30,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 587 msec 2024-12-05T12:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:30,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 608 msec 2024-12-05T12:13:30,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/6afbe5efcfc042e2b85e2c458a18611a is 50, key is test_row_0/A:col10/1733400810379/Put/seqid=0 2024-12-05T12:13:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T12:13:30,534 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:30,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-05T12:13:30,539 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T12:13:30,540 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:30,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:30,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:30,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400870550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400870546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:30,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741879_1055 (size=16931) 2024-12-05T12:13:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T12:13:30,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400870663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400870663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:30,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T12:13:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:30,814 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/e9544160d02c4dcea8fe0091951dd71c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e9544160d02c4dcea8fe0091951dd71c 2024-12-05T12:13:30,823 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into e9544160d02c4dcea8fe0091951dd71c(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:30,823 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:30,823 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=13, startTime=1733400810193; duration=0sec 2024-12-05T12:13:30,823 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:30,823 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:30,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T12:13:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:30,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T12:13:30,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400870873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:30,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:30,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400870873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:30,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/6afbe5efcfc042e2b85e2c458a18611a 2024-12-05T12:13:31,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/f585c0b3c1e2407f839c8dc46f5c3dd5 is 50, key is test_row_0/B:col10/1733400810379/Put/seqid=0 2024-12-05T12:13:31,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T12:13:31,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:31,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741880_1056 (size=12151) 2024-12-05T12:13:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T12:13:31,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T12:13:31,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:31,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400871183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:31,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400871183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:31,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T12:13:31,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:31,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400871333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:31,336 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:31,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400871352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:31,356 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:31,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/f585c0b3c1e2407f839c8dc46f5c3dd5 2024-12-05T12:13:31,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/faac1939b3a9483596091ddb09bfde94 is 50, key is test_row_0/C:col10/1733400810379/Put/seqid=0 2024-12-05T12:13:31,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741881_1057 (size=12151) 2024-12-05T12:13:31,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T12:13:31,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:31,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T12:13:31,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:31,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T12:13:31,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400871692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:31,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400871703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:31,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T12:13:31,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:31,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:31,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/faac1939b3a9483596091ddb09bfde94 2024-12-05T12:13:31,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/6afbe5efcfc042e2b85e2c458a18611a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6afbe5efcfc042e2b85e2c458a18611a 2024-12-05T12:13:31,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6afbe5efcfc042e2b85e2c458a18611a, entries=250, sequenceid=207, filesize=16.5 K 2024-12-05T12:13:31,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/f585c0b3c1e2407f839c8dc46f5c3dd5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f585c0b3c1e2407f839c8dc46f5c3dd5 2024-12-05T12:13:31,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f585c0b3c1e2407f839c8dc46f5c3dd5, entries=150, sequenceid=207, filesize=11.9 K 2024-12-05T12:13:31,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/faac1939b3a9483596091ddb09bfde94 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/faac1939b3a9483596091ddb09bfde94 2024-12-05T12:13:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1733400871863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:31,869 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:31,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/faac1939b3a9483596091ddb09bfde94, entries=150, sequenceid=207, filesize=11.9 K 2024-12-05T12:13:31,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9c2bbad557c305bbacb16e1dfd91bc9a in 1389ms, sequenceid=207, compaction requested=true 2024-12-05T12:13:31,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:31,888 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:31,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:31,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:31,888 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:31,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:31,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:31,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:31,890 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:31,890 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:31,890 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:31,890 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,890 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:31,890 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,890 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c0e6fc1e44424c8e9c0b604e18378101, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/80d9211931c64f5db96dab99dbac5395, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6afbe5efcfc042e2b85e2c458a18611a] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=40.7 K 2024-12-05T12:13:31,890 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f35c72c2710548de9f1c9a81916b6b37, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4962cf232b604fe9b39849086eebb162, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f585c0b3c1e2407f839c8dc46f5c3dd5] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=36.0 K 2024-12-05T12:13:31,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:31,891 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e6fc1e44424c8e9c0b604e18378101, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733400808967 2024-12-05T12:13:31,891 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting f35c72c2710548de9f1c9a81916b6b37, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733400808967 2024-12-05T12:13:31,892 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 4962cf232b604fe9b39849086eebb162, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733400809652 2024-12-05T12:13:31,892 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80d9211931c64f5db96dab99dbac5395, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733400809652 2024-12-05T12:13:31,892 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting f585c0b3c1e2407f839c8dc46f5c3dd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733400810379 2024-12-05T12:13:31,892 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6afbe5efcfc042e2b85e2c458a18611a, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733400810317 2024-12-05T12:13:31,917 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:31,918 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/3bc060a1a3144999a7aac2d6140a4e89 is 50, key is test_row_0/B:col10/1733400810379/Put/seqid=0 2024-12-05T12:13:31,929 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:31,930 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/c6efd996dde64477b8fc0772d9714960 is 50, key is test_row_0/A:col10/1733400810379/Put/seqid=0 2024-12-05T12:13:31,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T12:13:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:31,934 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T12:13:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:31,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/6f74d5d18fb04426ab5b022e0656b149 is 50, key is test_row_0/A:col10/1733400810544/Put/seqid=0 2024-12-05T12:13:31,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741882_1058 (size=12663) 2024-12-05T12:13:31,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741883_1059 (size=12663) 2024-12-05T12:13:31,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741884_1060 (size=12151) 2024-12-05T12:13:31,977 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/6f74d5d18fb04426ab5b022e0656b149 2024-12-05T12:13:31,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/177b7ab283404133a98916260ef51370 is 50, key is test_row_0/B:col10/1733400810544/Put/seqid=0 2024-12-05T12:13:31,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741885_1061 (size=12151) 2024-12-05T12:13:31,996 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/177b7ab283404133a98916260ef51370 2024-12-05T12:13:32,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/4cc1980a782641ce8d49dcea2078195c is 50, key is test_row_0/C:col10/1733400810544/Put/seqid=0 2024-12-05T12:13:32,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741886_1062 (size=12151) 2024-12-05T12:13:32,367 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/3bc060a1a3144999a7aac2d6140a4e89 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/3bc060a1a3144999a7aac2d6140a4e89 2024-12-05T12:13:32,385 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into 3bc060a1a3144999a7aac2d6140a4e89(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:32,385 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:32,385 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=13, startTime=1733400811888; duration=0sec 2024-12-05T12:13:32,385 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/c6efd996dde64477b8fc0772d9714960 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c6efd996dde64477b8fc0772d9714960 2024-12-05T12:13:32,385 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:32,386 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:32,386 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:32,391 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:32,391 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:32,391 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:32,391 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e9544160d02c4dcea8fe0091951dd71c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/a3d308c15f4743b9b5646fd09877fdaa, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/faac1939b3a9483596091ddb09bfde94] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=36.0 K 2024-12-05T12:13:32,392 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting e9544160d02c4dcea8fe0091951dd71c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733400808967 2024-12-05T12:13:32,393 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting a3d308c15f4743b9b5646fd09877fdaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733400809652 2024-12-05T12:13:32,393 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting faac1939b3a9483596091ddb09bfde94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733400810379 2024-12-05T12:13:32,395 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into c6efd996dde64477b8fc0772d9714960(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:32,395 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:32,395 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=13, startTime=1733400811887; duration=0sec 2024-12-05T12:13:32,395 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:32,395 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:32,409 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:32,409 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/716a01a3005f47fc894d98e99291fddd is 50, key is test_row_0/C:col10/1733400810379/Put/seqid=0 2024-12-05T12:13:32,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741887_1063 (size=12663) 2024-12-05T12:13:32,429 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/4cc1980a782641ce8d49dcea2078195c 2024-12-05T12:13:32,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/6f74d5d18fb04426ab5b022e0656b149 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6f74d5d18fb04426ab5b022e0656b149 2024-12-05T12:13:32,450 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6f74d5d18fb04426ab5b022e0656b149, entries=150, sequenceid=221, filesize=11.9 K 2024-12-05T12:13:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/177b7ab283404133a98916260ef51370 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/177b7ab283404133a98916260ef51370 2024-12-05T12:13:32,461 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/177b7ab283404133a98916260ef51370, entries=150, sequenceid=221, filesize=11.9 K 2024-12-05T12:13:32,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/4cc1980a782641ce8d49dcea2078195c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4cc1980a782641ce8d49dcea2078195c 2024-12-05T12:13:32,477 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4cc1980a782641ce8d49dcea2078195c, entries=150, sequenceid=221, filesize=11.9 K 2024-12-05T12:13:32,479 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 9c2bbad557c305bbacb16e1dfd91bc9a in 544ms, sequenceid=221, compaction requested=false 2024-12-05T12:13:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-05T12:13:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-05T12:13:32,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-05T12:13:32,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-12-05T12:13:32,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.9530 sec 2024-12-05T12:13:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T12:13:32,683 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:32,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-05T12:13:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T12:13:32,687 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:32,691 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:32,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:32,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:13:32,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:32,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:32,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:32,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:32,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:32,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:32,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/9ec19e68c86c45c3bd3b5df396837e38 is 50, key is test_row_0/A:col10/1733400812734/Put/seqid=0 2024-12-05T12:13:32,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741888_1064 (size=12151) 2024-12-05T12:13:32,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/9ec19e68c86c45c3bd3b5df396837e38 2024-12-05T12:13:32,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/8debc5bd2540451690e94cbd1b2f2c5a is 50, key is test_row_0/B:col10/1733400812734/Put/seqid=0 2024-12-05T12:13:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T12:13:32,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400812776, max timestamp=1733400812792Num uniques 859; 2024-12-05T12:13:32,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:32,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400812776, max timestamp=1733400812792Num uniques 859; 2024-12-05T12:13:32,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:32,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400812776, max timestamp=1733400812792Num uniques 859; 2024-12-05T12:13:32,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:32,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741889_1065 (size=12151) 2024-12-05T12:13:32,841 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/716a01a3005f47fc894d98e99291fddd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/716a01a3005f47fc894d98e99291fddd 2024-12-05T12:13:32,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/8debc5bd2540451690e94cbd1b2f2c5a 2024-12-05T12:13:32,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:32,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:32,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:32,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:32,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:32,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:32,855 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into 716a01a3005f47fc894d98e99291fddd(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:32,855 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:32,855 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=13, startTime=1733400811889; duration=0sec 2024-12-05T12:13:32,856 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:32,856 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:32,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/76961b1fbe7b43b48a94789524ae1c76 is 50, key is test_row_0/C:col10/1733400812734/Put/seqid=0 2024-12-05T12:13:32,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400872855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:32,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:32,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:32,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400872861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:32,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741890_1066 (size=12151) 2024-12-05T12:13:32,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/76961b1fbe7b43b48a94789524ae1c76 2024-12-05T12:13:32,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/9ec19e68c86c45c3bd3b5df396837e38 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9ec19e68c86c45c3bd3b5df396837e38 2024-12-05T12:13:32,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9ec19e68c86c45c3bd3b5df396837e38, entries=150, sequenceid=233, filesize=11.9 K 2024-12-05T12:13:32,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/8debc5bd2540451690e94cbd1b2f2c5a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8debc5bd2540451690e94cbd1b2f2c5a 2024-12-05T12:13:32,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8debc5bd2540451690e94cbd1b2f2c5a, entries=150, sequenceid=233, filesize=11.9 K 2024-12-05T12:13:32,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/76961b1fbe7b43b48a94789524ae1c76 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/76961b1fbe7b43b48a94789524ae1c76 2024-12-05T12:13:32,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/76961b1fbe7b43b48a94789524ae1c76, entries=150, sequenceid=233, filesize=11.9 K 2024-12-05T12:13:32,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9c2bbad557c305bbacb16e1dfd91bc9a in 192ms, sequenceid=233, compaction requested=true 2024-12-05T12:13:32,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:32,927 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:32,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:32,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:32,928 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:32,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:32,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:32,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:32,930 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:32,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:32,930 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:32,930 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:32,931 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:32,931 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:32,931 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/3bc060a1a3144999a7aac2d6140a4e89, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/177b7ab283404133a98916260ef51370, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8debc5bd2540451690e94cbd1b2f2c5a] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=36.1 K 2024-12-05T12:13:32,931 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:32,931 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c6efd996dde64477b8fc0772d9714960, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6f74d5d18fb04426ab5b022e0656b149, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9ec19e68c86c45c3bd3b5df396837e38] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=36.1 K 2024-12-05T12:13:32,931 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc060a1a3144999a7aac2d6140a4e89, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733400810379 2024-12-05T12:13:32,932 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6efd996dde64477b8fc0772d9714960, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733400810379 2024-12-05T12:13:32,932 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 177b7ab283404133a98916260ef51370, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733400810535 2024-12-05T12:13:32,933 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f74d5d18fb04426ab5b022e0656b149, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733400810535 2024-12-05T12:13:32,933 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 8debc5bd2540451690e94cbd1b2f2c5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733400812730 2024-12-05T12:13:32,934 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ec19e68c86c45c3bd3b5df396837e38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733400812730 2024-12-05T12:13:32,953 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:32,954 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/d86823476e8d4ef9b7321e62214ea896 is 50, key is test_row_0/B:col10/1733400812734/Put/seqid=0 2024-12-05T12:13:32,956 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:32,956 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/823b587f72d54d2dab129d119af44038 is 50, key is test_row_0/A:col10/1733400812734/Put/seqid=0 2024-12-05T12:13:32,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741892_1068 (size=12765) 2024-12-05T12:13:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:32,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733400812828, max timestamp=1733400812975Num uniques 1018; 2024-12-05T12:13:32,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7685, totalHeapSize=17712, min timestamp=1733400812828, max timestamp=1733400812977Num uniques 1018; 2024-12-05T12:13:32,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46615, new segment data size=7685, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T12:13:32,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=71.56 KB heapSize=198.72 KB 2024-12-05T12:13:32,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:32,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:13:32,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:32,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:32,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:32,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:32,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T12:13:32,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741891_1067 (size=12765) 2024-12-05T12:13:32,997 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/823b587f72d54d2dab129d119af44038 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/823b587f72d54d2dab129d119af44038 2024-12-05T12:13:33,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T12:13:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/30693d5314134472a6100136dcc3c42b is 50, key is test_row_0/A:col10/1733400812975/Put/seqid=0 2024-12-05T12:13:33,013 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/d86823476e8d4ef9b7321e62214ea896 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/d86823476e8d4ef9b7321e62214ea896 2024-12-05T12:13:33,023 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into 823b587f72d54d2dab129d119af44038(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:33,023 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:33,024 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=13, startTime=1733400812927; duration=0sec 2024-12-05T12:13:33,024 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:33,025 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:33,025 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:33,026 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:33,027 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:33,027 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into d86823476e8d4ef9b7321e62214ea896(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:33,027 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,027 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:33,027 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=13, startTime=1733400812928; duration=0sec 2024-12-05T12:13:33,027 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/716a01a3005f47fc894d98e99291fddd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4cc1980a782641ce8d49dcea2078195c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/76961b1fbe7b43b48a94789524ae1c76] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=36.1 K 2024-12-05T12:13:33,027 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:33,027 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:33,027 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 716a01a3005f47fc894d98e99291fddd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733400810379 2024-12-05T12:13:33,028 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cc1980a782641ce8d49dcea2078195c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733400810535 2024-12-05T12:13:33,028 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76961b1fbe7b43b48a94789524ae1c76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733400812730 2024-12-05T12:13:33,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741893_1069 (size=12301) 2024-12-05T12:13:33,043 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:33,044 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/00d2c0ec4aeb4b55bb882e6aa7aa07b6 is 50, key is test_row_0/C:col10/1733400812734/Put/seqid=0 2024-12-05T12:13:33,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741894_1070 (size=12765) 2024-12-05T12:13:33,090 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/00d2c0ec4aeb4b55bb882e6aa7aa07b6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/00d2c0ec4aeb4b55bb882e6aa7aa07b6 2024-12-05T12:13:33,110 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into 00d2c0ec4aeb4b55bb882e6aa7aa07b6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:33,111 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:33,111 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=13, startTime=1733400812928; duration=0sec 2024-12-05T12:13:33,111 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:33,111 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:33,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:33,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400873103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400873103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:33,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:33,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:33,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400873223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:33,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400873224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:33,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:33,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T12:13:33,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/30693d5314134472a6100136dcc3c42b 2024-12-05T12:13:33,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400873433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:33,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/8e1bcfaf2da74cd2b883f604de5e41fd is 50, key is test_row_0/B:col10/1733400812975/Put/seqid=0 2024-12-05T12:13:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400873443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:33,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741895_1071 (size=12301) 2024-12-05T12:13:33,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:33,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:33,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:33,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:33,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400873742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:33,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:33,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400873753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:33,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:33,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T12:13:33,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/8e1bcfaf2da74cd2b883f604de5e41fd 2024-12-05T12:13:33,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ad93db5180444ebe9251f992290030e4 is 50, key is test_row_0/C:col10/1733400812975/Put/seqid=0 2024-12-05T12:13:33,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741896_1072 (size=12301) 2024-12-05T12:13:33,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:33,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:33,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:33,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:34,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:34,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:34,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:34,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:34,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:34,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:34,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:34,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:34,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:34,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:34,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400874252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:34,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:34,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400874263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:34,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ad93db5180444ebe9251f992290030e4 2024-12-05T12:13:34,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:34,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:34,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:34,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/30693d5314134472a6100136dcc3c42b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/30693d5314134472a6100136dcc3c42b 2024-12-05T12:13:34,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/30693d5314134472a6100136dcc3c42b, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T12:13:34,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/8e1bcfaf2da74cd2b883f604de5e41fd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8e1bcfaf2da74cd2b883f604de5e41fd 2024-12-05T12:13:34,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8e1bcfaf2da74cd2b883f604de5e41fd, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T12:13:34,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ad93db5180444ebe9251f992290030e4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ad93db5180444ebe9251f992290030e4 2024-12-05T12:13:34,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ad93db5180444ebe9251f992290030e4, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T12:13:34,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=114.05 KB/116790 for 9c2bbad557c305bbacb16e1dfd91bc9a in 1321ms, sequenceid=273, compaction requested=false 2024-12-05T12:13:34,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:34,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T12:13:34,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:34,407 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T12:13:34,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:34,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:34,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:34,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:34,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:34,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:34,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/055601fa9a2f48e991c969ea1a6bf34a is 50, key is test_row_0/A:col10/1733400813100/Put/seqid=0 2024-12-05T12:13:34,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741897_1073 (size=12301) 2024-12-05T12:13:34,428 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/055601fa9a2f48e991c969ea1a6bf34a 2024-12-05T12:13:34,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/995cc6cfefd44a5aa06890f4101171ef is 50, key is test_row_0/B:col10/1733400813100/Put/seqid=0 2024-12-05T12:13:34,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741898_1074 (size=12301) 2024-12-05T12:13:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T12:13:34,852 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/995cc6cfefd44a5aa06890f4101171ef 2024-12-05T12:13:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/74551749a0d74f628a3bd711227b0c6a is 50, key is test_row_0/C:col10/1733400813100/Put/seqid=0 2024-12-05T12:13:34,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741899_1075 (size=12301) 2024-12-05T12:13:35,267 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/74551749a0d74f628a3bd711227b0c6a 2024-12-05T12:13:35,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/055601fa9a2f48e991c969ea1a6bf34a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/055601fa9a2f48e991c969ea1a6bf34a 2024-12-05T12:13:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:35,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:35,293 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/055601fa9a2f48e991c969ea1a6bf34a, entries=150, sequenceid=296, filesize=12.0 K 2024-12-05T12:13:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/995cc6cfefd44a5aa06890f4101171ef as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/995cc6cfefd44a5aa06890f4101171ef 2024-12-05T12:13:35,308 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/995cc6cfefd44a5aa06890f4101171ef, entries=150, sequenceid=296, filesize=12.0 K 2024-12-05T12:13:35,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/74551749a0d74f628a3bd711227b0c6a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/74551749a0d74f628a3bd711227b0c6a 2024-12-05T12:13:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400875311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400875318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,325 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/74551749a0d74f628a3bd711227b0c6a, entries=150, sequenceid=296, filesize=12.0 K 2024-12-05T12:13:35,335 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9c2bbad557c305bbacb16e1dfd91bc9a in 928ms, sequenceid=296, compaction requested=true 2024-12-05T12:13:35,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:35,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:35,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-05T12:13:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-05T12:13:35,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-05T12:13:35,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6470 sec 2024-12-05T12:13:35,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.6650 sec 2024-12-05T12:13:35,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T12:13:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:35,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:35,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:35,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/a8ce92768d3b498e9a37c25a6a4f56bd is 50, key is test_row_0/A:col10/1733400815350/Put/seqid=0 2024-12-05T12:13:35,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741900_1076 (size=12301) 2024-12-05T12:13:35,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/a8ce92768d3b498e9a37c25a6a4f56bd 2024-12-05T12:13:35,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/0e22ff7d8d5c4d748d63966e4154da49 is 50, key is test_row_0/B:col10/1733400815350/Put/seqid=0 2024-12-05T12:13:35,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400875426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400875431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400875433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400875436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741901_1077 (size=12301) 2024-12-05T12:13:35,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/0e22ff7d8d5c4d748d63966e4154da49 2024-12-05T12:13:35,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/215b510517344e0582818be13fe140b3 is 50, key is test_row_0/C:col10/1733400815350/Put/seqid=0 2024-12-05T12:13:35,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741902_1078 (size=12301) 2024-12-05T12:13:35,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400875543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400875543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400875554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400875654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400875753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400875754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400875763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/215b510517344e0582818be13fe140b3 2024-12-05T12:13:35,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/a8ce92768d3b498e9a37c25a6a4f56bd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a8ce92768d3b498e9a37c25a6a4f56bd 2024-12-05T12:13:35,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400875964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:35,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a8ce92768d3b498e9a37c25a6a4f56bd, entries=150, sequenceid=309, filesize=12.0 K 2024-12-05T12:13:35,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/0e22ff7d8d5c4d748d63966e4154da49 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/0e22ff7d8d5c4d748d63966e4154da49 2024-12-05T12:13:35,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/0e22ff7d8d5c4d748d63966e4154da49, entries=150, sequenceid=309, filesize=12.0 K 2024-12-05T12:13:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/215b510517344e0582818be13fe140b3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/215b510517344e0582818be13fe140b3 2024-12-05T12:13:36,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/215b510517344e0582818be13fe140b3, entries=150, sequenceid=309, filesize=12.0 K 2024-12-05T12:13:36,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 9c2bbad557c305bbacb16e1dfd91bc9a in 649ms, sequenceid=309, compaction requested=true 2024-12-05T12:13:36,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:36,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:36,003 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:13:36,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:36,003 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:13:36,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:36,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:36,005 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:13:36,005 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:13:36,005 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:36,005 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:36,005 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:36,005 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:36,005 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/823b587f72d54d2dab129d119af44038, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/30693d5314134472a6100136dcc3c42b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/055601fa9a2f48e991c969ea1a6bf34a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a8ce92768d3b498e9a37c25a6a4f56bd] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=48.5 K 2024-12-05T12:13:36,005 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/d86823476e8d4ef9b7321e62214ea896, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8e1bcfaf2da74cd2b883f604de5e41fd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/995cc6cfefd44a5aa06890f4101171ef, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/0e22ff7d8d5c4d748d63966e4154da49] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=48.5 K 2024-12-05T12:13:36,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:36,006 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d86823476e8d4ef9b7321e62214ea896, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733400812730 2024-12-05T12:13:36,006 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 823b587f72d54d2dab129d119af44038, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733400812730 2024-12-05T12:13:36,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:36,007 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30693d5314134472a6100136dcc3c42b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733400812834 2024-12-05T12:13:36,007 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e1bcfaf2da74cd2b883f604de5e41fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733400812834 2024-12-05T12:13:36,007 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 055601fa9a2f48e991c969ea1a6bf34a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733400813043 2024-12-05T12:13:36,007 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 995cc6cfefd44a5aa06890f4101171ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733400813043 2024-12-05T12:13:36,007 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e22ff7d8d5c4d748d63966e4154da49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733400815298 2024-12-05T12:13:36,007 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8ce92768d3b498e9a37c25a6a4f56bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733400815298 2024-12-05T12:13:36,083 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:36,084 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/54ce1469c9be4409bdac0ec283bc3bed is 50, key is test_row_0/B:col10/1733400815350/Put/seqid=0 2024-12-05T12:13:36,083 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:36,087 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/cb9d38c57a92415dbe30e6f991e66037 is 50, key is test_row_0/A:col10/1733400815350/Put/seqid=0 2024-12-05T12:13:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400815414, max timestamp=1733400816082Num uniques 859; 2024-12-05T12:13:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400815414, max timestamp=1733400816082Num uniques 859; 2024-12-05T12:13:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:36,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-05T12:13:36,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733400815414, max timestamp=1733400816083Num uniques 859; 2024-12-05T12:13:36,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T12:13:36,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:36,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:36,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:36,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:36,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:36,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:36,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/b99ec3abb56a469eadbab6a5120b5477 is 50, key is test_row_0/A:col10/1733400816084/Put/seqid=0 2024-12-05T12:13:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741904_1080 (size=13051) 2024-12-05T12:13:36,164 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/cb9d38c57a92415dbe30e6f991e66037 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/cb9d38c57a92415dbe30e6f991e66037 2024-12-05T12:13:36,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741903_1079 (size=13051) 2024-12-05T12:13:36,180 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into cb9d38c57a92415dbe30e6f991e66037(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:36,180 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:36,180 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=12, startTime=1733400816003; duration=0sec 2024-12-05T12:13:36,181 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:36,181 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:36,181 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:13:36,183 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:13:36,184 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:36,184 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:36,184 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/00d2c0ec4aeb4b55bb882e6aa7aa07b6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ad93db5180444ebe9251f992290030e4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/74551749a0d74f628a3bd711227b0c6a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/215b510517344e0582818be13fe140b3] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=48.5 K 2024-12-05T12:13:36,186 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00d2c0ec4aeb4b55bb882e6aa7aa07b6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733400812730 2024-12-05T12:13:36,186 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad93db5180444ebe9251f992290030e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733400812834 2024-12-05T12:13:36,187 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74551749a0d74f628a3bd711227b0c6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733400813043 2024-12-05T12:13:36,188 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 215b510517344e0582818be13fe140b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733400815298 2024-12-05T12:13:36,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741905_1081 (size=12301) 2024-12-05T12:13:36,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400816146, max timestamp=1733400816194Num uniques 859; 2024-12-05T12:13:36,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400816146, max timestamp=1733400816194Num uniques 859; 2024-12-05T12:13:36,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:36,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:36,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400816146, max timestamp=1733400816194Num uniques 859; 2024-12-05T12:13:36,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:36,234 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:36,235 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/796b85db627e4c538648b901c461d3c7 is 50, key is test_row_0/C:col10/1733400815350/Put/seqid=0 2024-12-05T12:13:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741906_1082 (size=13051) 2024-12-05T12:13:36,273 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/796b85db627e4c538648b901c461d3c7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/796b85db627e4c538648b901c461d3c7 2024-12-05T12:13:36,284 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into 796b85db627e4c538648b901c461d3c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:36,284 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:36,284 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=12, startTime=1733400816004; duration=0sec 2024-12-05T12:13:36,285 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:36,285 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:36,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733400816292, max timestamp=1733400816317Num uniques 1018; 2024-12-05T12:13:36,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-05T12:13:36,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733400816302, max timestamp=1733400816317Num uniques 1018; 2024-12-05T12:13:36,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733400816302, max timestamp=1733400816317Num uniques 1018; 2024-12-05T12:13:36,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T12:13:36,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-05T12:13:36,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733400816352, max timestamp=1733400816374Num uniques 1177; 2024-12-05T12:13:36,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-05T12:13:36,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733400816353, max timestamp=1733400816374Num uniques 1027; 2024-12-05T12:13:36,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-05T12:13:36,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733400816353, max timestamp=1733400816374Num uniques 1009; 2024-12-05T12:13:36,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-12-05T12:13:36,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733400816394, max timestamp=1733400816420Num uniques 1136; 2024-12-05T12:13:36,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-12-05T12:13:36,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733400816394, max timestamp=1733400816420Num uniques 1036; 2024-12-05T12:13:36,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-12-05T12:13:36,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733400816394, max timestamp=1733400816420Num uniques 1036; 2024-12-05T12:13:36,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-12-05T12:13:36,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400876462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400876474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400876474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400876474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400876583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400876583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400876583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,596 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/54ce1469c9be4409bdac0ec283bc3bed as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/54ce1469c9be4409bdac0ec283bc3bed 2024-12-05T12:13:36,607 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into 54ce1469c9be4409bdac0ec283bc3bed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:36,607 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:36,607 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=12, startTime=1733400816003; duration=0sec 2024-12-05T12:13:36,608 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:36,608 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:36,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/b99ec3abb56a469eadbab6a5120b5477 2024-12-05T12:13:36,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/a40154a144694065b28dae9b070d86a2 is 50, key is test_row_0/B:col10/1733400816084/Put/seqid=0 2024-12-05T12:13:36,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741907_1083 (size=12301) 2024-12-05T12:13:36,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/a40154a144694065b28dae9b070d86a2 2024-12-05T12:13:36,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/0ebc6dd2c20c4116ab3e659494929616 is 50, key is test_row_0/C:col10/1733400816084/Put/seqid=0 2024-12-05T12:13:36,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741908_1084 (size=12301) 2024-12-05T12:13:36,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/0ebc6dd2c20c4116ab3e659494929616 2024-12-05T12:13:36,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/b99ec3abb56a469eadbab6a5120b5477 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b99ec3abb56a469eadbab6a5120b5477 2024-12-05T12:13:36,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b99ec3abb56a469eadbab6a5120b5477, entries=150, sequenceid=332, filesize=12.0 K 2024-12-05T12:13:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/a40154a144694065b28dae9b070d86a2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/a40154a144694065b28dae9b070d86a2 2024-12-05T12:13:36,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/a40154a144694065b28dae9b070d86a2, entries=150, sequenceid=332, filesize=12.0 K 2024-12-05T12:13:36,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/0ebc6dd2c20c4116ab3e659494929616 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/0ebc6dd2c20c4116ab3e659494929616 2024-12-05T12:13:36,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/0ebc6dd2c20c4116ab3e659494929616, entries=150, sequenceid=332, filesize=12.0 K 2024-12-05T12:13:36,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.86 KB/40815, heapSize ~105.12 KB/107640, currentSize=147.60 KB/151140 for 9c2bbad557c305bbacb16e1dfd91bc9a in 690ms, sequenceid=332, compaction requested=false 2024-12-05T12:13:36,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:36,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=154.31 KB heapSize=428.16 KB 2024-12-05T12:13:36,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:36,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:13:36,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:36,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:13:36,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:36,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:13:36,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/23073b766d0f490cae82cecdf1baba17 is 50, key is test_row_0/A:col10/1733400816468/Put/seqid=0 2024-12-05T12:13:36,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400876818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400876818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400876823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741909_1085 (size=17181) 2024-12-05T12:13:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T12:13:36,854 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:36,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-05T12:13:36,861 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T12:13:36,862 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:36,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:36,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400876933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400876933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400876933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T12:13:37,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T12:13:37,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:37,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400877143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:37,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400877143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:37,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400877143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:37,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T12:13:37,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:37,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T12:13:37,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/23073b766d0f490cae82cecdf1baba17 2024-12-05T12:13:37,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7990048fd36140ddae7862cb48f1ba77 is 50, key is test_row_0/B:col10/1733400816468/Put/seqid=0 2024-12-05T12:13:37,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741910_1086 (size=12301) 2024-12-05T12:13:37,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7990048fd36140ddae7862cb48f1ba77 2024-12-05T12:13:37,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/e1c5e5415bd641178f962a5621392685 is 50, key is test_row_0/C:col10/1733400816468/Put/seqid=0 2024-12-05T12:13:37,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T12:13:37,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:37,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741911_1087 (size=12301) 2024-12-05T12:13:37,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400877453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:37,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400877453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400877453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:37,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T12:13:37,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:37,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T12:13:37,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400877493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:37,499 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:37,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T12:13:37,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:37,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:37,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/e1c5e5415bd641178f962a5621392685 2024-12-05T12:13:37,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/23073b766d0f490cae82cecdf1baba17 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/23073b766d0f490cae82cecdf1baba17 2024-12-05T12:13:37,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/23073b766d0f490cae82cecdf1baba17, entries=250, sequenceid=424, filesize=16.8 K 2024-12-05T12:13:37,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7990048fd36140ddae7862cb48f1ba77 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7990048fd36140ddae7862cb48f1ba77 2024-12-05T12:13:37,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7990048fd36140ddae7862cb48f1ba77, entries=150, sequenceid=424, filesize=12.0 K 2024-12-05T12:13:37,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/e1c5e5415bd641178f962a5621392685 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e1c5e5415bd641178f962a5621392685 2024-12-05T12:13:37,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e1c5e5415bd641178f962a5621392685, entries=150, sequenceid=424, filesize=12.0 K 2024-12-05T12:13:37,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~448.73 KB/459504, currentSize=26.84 KB/27480 for 9c2bbad557c305bbacb16e1dfd91bc9a in 985ms, sequenceid=424, compaction requested=true 2024-12-05T12:13:37,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:37,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:37,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:37,791 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:37,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:37,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:37,791 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:37,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:37,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:37,792 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:37,792 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:37,792 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:37,792 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,792 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:37,792 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,792 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/54ce1469c9be4409bdac0ec283bc3bed, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/a40154a144694065b28dae9b070d86a2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7990048fd36140ddae7862cb48f1ba77] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=36.8 K 2024-12-05T12:13:37,792 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/cb9d38c57a92415dbe30e6f991e66037, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b99ec3abb56a469eadbab6a5120b5477, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/23073b766d0f490cae82cecdf1baba17] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=41.5 K 2024-12-05T12:13:37,793 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ce1469c9be4409bdac0ec283bc3bed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733400815298 2024-12-05T12:13:37,793 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb9d38c57a92415dbe30e6f991e66037, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733400815298 2024-12-05T12:13:37,793 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting a40154a144694065b28dae9b070d86a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733400815428 2024-12-05T12:13:37,793 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b99ec3abb56a469eadbab6a5120b5477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733400815428 2024-12-05T12:13:37,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T12:13:37,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,794 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-05T12:13:37,794 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 7990048fd36140ddae7862cb48f1ba77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733400816468 2024-12-05T12:13:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:37,795 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23073b766d0f490cae82cecdf1baba17, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733400816458 2024-12-05T12:13:37,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/b6723cb1b1f7479295bdf78b74ec4909 is 50, key is test_row_0/A:col10/1733400816817/Put/seqid=0 2024-12-05T12:13:37,808 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:37,809 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/c8a788b5300244f7b26105dfdbfcf451 is 50, key is test_row_0/A:col10/1733400816468/Put/seqid=0 2024-12-05T12:13:37,816 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:37,817 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/8cead26d29cc49c794356de39ed16d51 is 50, key is test_row_0/B:col10/1733400816468/Put/seqid=0 2024-12-05T12:13:37,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741912_1088 (size=12301) 2024-12-05T12:13:37,868 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/b6723cb1b1f7479295bdf78b74ec4909 2024-12-05T12:13:37,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741913_1089 (size=13153) 2024-12-05T12:13:37,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741914_1090 (size=13153) 2024-12-05T12:13:37,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/38ec016ac21f4ef987d8cbcd70cce6b0 is 50, key is test_row_0/B:col10/1733400816817/Put/seqid=0 2024-12-05T12:13:37,896 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/8cead26d29cc49c794356de39ed16d51 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8cead26d29cc49c794356de39ed16d51 2024-12-05T12:13:37,907 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into 8cead26d29cc49c794356de39ed16d51(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:37,907 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:37,908 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=13, startTime=1733400817791; duration=0sec 2024-12-05T12:13:37,909 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:37,909 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:37,909 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:37,910 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:37,910 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:37,911 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:37,911 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/796b85db627e4c538648b901c461d3c7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/0ebc6dd2c20c4116ab3e659494929616, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e1c5e5415bd641178f962a5621392685] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=36.8 K 2024-12-05T12:13:37,911 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 796b85db627e4c538648b901c461d3c7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733400815298 2024-12-05T12:13:37,912 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ebc6dd2c20c4116ab3e659494929616, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733400815428 2024-12-05T12:13:37,912 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting e1c5e5415bd641178f962a5621392685, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733400816468 2024-12-05T12:13:37,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741915_1091 (size=12301) 2024-12-05T12:13:37,923 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/38ec016ac21f4ef987d8cbcd70cce6b0 2024-12-05T12:13:37,931 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:37,932 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/18ec9be7fd7043ea8acb8e85813035c3 is 50, key is test_row_0/C:col10/1733400816468/Put/seqid=0 2024-12-05T12:13:37,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/fba26d331aac44a8a6885725d952f674 is 50, key is test_row_0/C:col10/1733400816817/Put/seqid=0 2024-12-05T12:13:37,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741917_1093 (size=13153) 2024-12-05T12:13:37,970 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/18ec9be7fd7043ea8acb8e85813035c3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/18ec9be7fd7043ea8acb8e85813035c3 2024-12-05T12:13:37,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741916_1092 (size=12301) 2024-12-05T12:13:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:37,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:37,977 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/fba26d331aac44a8a6885725d952f674 2024-12-05T12:13:37,985 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into 18ec9be7fd7043ea8acb8e85813035c3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:37,985 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:37,985 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=13, startTime=1733400817791; duration=0sec 2024-12-05T12:13:37,985 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:37,985 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:37,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/b6723cb1b1f7479295bdf78b74ec4909 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b6723cb1b1f7479295bdf78b74ec4909 2024-12-05T12:13:37,999 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b6723cb1b1f7479295bdf78b74ec4909, entries=150, sequenceid=431, filesize=12.0 K 2024-12-05T12:13:38,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/38ec016ac21f4ef987d8cbcd70cce6b0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/38ec016ac21f4ef987d8cbcd70cce6b0 2024-12-05T12:13:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T12:13:38,013 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/38ec016ac21f4ef987d8cbcd70cce6b0, entries=150, sequenceid=431, filesize=12.0 K 2024-12-05T12:13:38,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/fba26d331aac44a8a6885725d952f674 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/fba26d331aac44a8a6885725d952f674 2024-12-05T12:13:38,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400817993, max timestamp=1733400818020Num uniques 859; 2024-12-05T12:13:38,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:38,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400817993, max timestamp=1733400818020Num uniques 859; 2024-12-05T12:13:38,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:38,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400817993, max timestamp=1733400818020Num uniques 859; 2024-12-05T12:13:38,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:13:38,026 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/fba26d331aac44a8a6885725d952f674, entries=150, sequenceid=431, filesize=12.0 K 2024-12-05T12:13:38,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=33.54 KB/34350 for 9c2bbad557c305bbacb16e1dfd91bc9a in 233ms, sequenceid=431, compaction requested=false 2024-12-05T12:13:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-05T12:13:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-05T12:13:38,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-05T12:13:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:38,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-05T12:13:38,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1690 sec 2024-12-05T12:13:38,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:38,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:38,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:38,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:38,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:38,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:38,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1840 sec 2024-12-05T12:13:38,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/a2f2e3650cdf4bcc8660cfe087f2433f is 50, key is test_row_0/A:col10/1733400818033/Put/seqid=0 2024-12-05T12:13:38,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741918_1094 (size=17181) 2024-12-05T12:13:38,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/a2f2e3650cdf4bcc8660cfe087f2433f 2024-12-05T12:13:38,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733400818078, max timestamp=1733400818090Num uniques 859; 2024-12-05T12:13:38,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:13:38,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733400818078, max timestamp=1733400818090Num uniques 859; 2024-12-05T12:13:38,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:13:38,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400818078, max timestamp=1733400818090Num uniques 859; 2024-12-05T12:13:38,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:13:38,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/9b70ff431bcf472b8a10949e5a8cbb69 is 50, key is test_row_0/B:col10/1733400818033/Put/seqid=0 2024-12-05T12:13:38,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741919_1095 (size=12301) 2024-12-05T12:13:38,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400878168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400878168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400878190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,283 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/c8a788b5300244f7b26105dfdbfcf451 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c8a788b5300244f7b26105dfdbfcf451 2024-12-05T12:13:38,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400878283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400878283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,294 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into c8a788b5300244f7b26105dfdbfcf451(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:38,294 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:38,294 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=13, startTime=1733400817790; duration=0sec 2024-12-05T12:13:38,294 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:38,294 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:38,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400878303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400878493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400878493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/9b70ff431bcf472b8a10949e5a8cbb69 2024-12-05T12:13:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400878513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/837e0a20635646e8853f603296ec9b3a is 50, key is test_row_0/C:col10/1733400818033/Put/seqid=0 2024-12-05T12:13:38,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741920_1096 (size=12301) 2024-12-05T12:13:38,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400878803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400878803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400878823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:38,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/837e0a20635646e8853f603296ec9b3a 2024-12-05T12:13:38,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/a2f2e3650cdf4bcc8660cfe087f2433f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a2f2e3650cdf4bcc8660cfe087f2433f 2024-12-05T12:13:38,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a2f2e3650cdf4bcc8660cfe087f2433f, entries=250, sequenceid=458, filesize=16.8 K 2024-12-05T12:13:38,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/9b70ff431bcf472b8a10949e5a8cbb69 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/9b70ff431bcf472b8a10949e5a8cbb69 2024-12-05T12:13:38,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/9b70ff431bcf472b8a10949e5a8cbb69, entries=150, sequenceid=458, filesize=12.0 K 2024-12-05T12:13:38,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/837e0a20635646e8853f603296ec9b3a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/837e0a20635646e8853f603296ec9b3a 2024-12-05T12:13:38,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/837e0a20635646e8853f603296ec9b3a, entries=150, sequenceid=458, filesize=12.0 K 2024-12-05T12:13:38,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 9c2bbad557c305bbacb16e1dfd91bc9a in 948ms, sequenceid=458, compaction requested=true 2024-12-05T12:13:38,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:38,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:38,987 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:38,987 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:38,988 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:38,988 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:38,988 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:38,988 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8cead26d29cc49c794356de39ed16d51, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/38ec016ac21f4ef987d8cbcd70cce6b0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/9b70ff431bcf472b8a10949e5a8cbb69] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=36.9 K 2024-12-05T12:13:38,989 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:38,989 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:38,989 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:38,989 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c8a788b5300244f7b26105dfdbfcf451, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b6723cb1b1f7479295bdf78b74ec4909, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a2f2e3650cdf4bcc8660cfe087f2433f] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=41.6 K 2024-12-05T12:13:38,990 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting c8a788b5300244f7b26105dfdbfcf451, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733400816468 2024-12-05T12:13:38,990 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cead26d29cc49c794356de39ed16d51, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733400816468 2024-12-05T12:13:38,990 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b6723cb1b1f7479295bdf78b74ec4909, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733400816814 2024-12-05T12:13:38,990 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38ec016ac21f4ef987d8cbcd70cce6b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733400816814 2024-12-05T12:13:38,991 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f2e3650cdf4bcc8660cfe087f2433f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733400818013 2024-12-05T12:13:38,991 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b70ff431bcf472b8a10949e5a8cbb69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733400818020 2024-12-05T12:13:39,005 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:39,006 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/924785558ea14fc4b5e02796385a65fb is 50, key is test_row_0/B:col10/1733400818033/Put/seqid=0 2024-12-05T12:13:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T12:13:39,023 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:39,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:39,025 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-05T12:13:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T12:13:39,026 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/9918098fc3e740828b9322fb626c23e0 is 50, key is test_row_0/A:col10/1733400818033/Put/seqid=0 2024-12-05T12:13:39,029 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:39,032 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:39,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:39,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741921_1097 (size=13255) 2024-12-05T12:13:39,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741922_1098 (size=13255) 2024-12-05T12:13:39,076 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/9918098fc3e740828b9322fb626c23e0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9918098fc3e740828b9322fb626c23e0 2024-12-05T12:13:39,085 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into 9918098fc3e740828b9322fb626c23e0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:39,085 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:39,085 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=13, startTime=1733400818986; duration=0sec 2024-12-05T12:13:39,086 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:39,086 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:39,086 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:39,089 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:39,090 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:39,090 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:39,093 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/18ec9be7fd7043ea8acb8e85813035c3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/fba26d331aac44a8a6885725d952f674, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/837e0a20635646e8853f603296ec9b3a] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=36.9 K 2024-12-05T12:13:39,094 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ec9be7fd7043ea8acb8e85813035c3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733400816468 2024-12-05T12:13:39,094 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting fba26d331aac44a8a6885725d952f674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733400816814 2024-12-05T12:13:39,094 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 837e0a20635646e8853f603296ec9b3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733400818020 2024-12-05T12:13:39,118 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:39,119 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/8b1d214576e74d779593cf4598541063 is 50, key is test_row_0/C:col10/1733400818033/Put/seqid=0 2024-12-05T12:13:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T12:13:39,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741923_1099 (size=13255) 2024-12-05T12:13:39,143 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/8b1d214576e74d779593cf4598541063 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/8b1d214576e74d779593cf4598541063 2024-12-05T12:13:39,159 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into 8b1d214576e74d779593cf4598541063(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:39,159 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:39,159 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=13, startTime=1733400818987; duration=0sec 2024-12-05T12:13:39,160 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:39,160 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:39,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T12:13:39,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:39,187 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-05T12:13:39,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:39,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:39,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:39,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:39,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:39,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:13:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/27fda8e44ca9484882be819ac4d268e2 is 50, key is test_row_0/A:col10/1733400818170/Put/seqid=0 2024-12-05T12:13:39,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741924_1100 (size=12301) 2024-12-05T12:13:39,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T12:13:39,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400879353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:39,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400879359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400879360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:39,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:39,460 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/924785558ea14fc4b5e02796385a65fb as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/924785558ea14fc4b5e02796385a65fb 2024-12-05T12:13:39,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400879463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:39,473 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into 924785558ea14fc4b5e02796385a65fb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:39,473 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:39,474 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=13, startTime=1733400818987; duration=0sec 2024-12-05T12:13:39,474 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:39,474 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:39,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400879474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400879474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:39,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400879513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:39,515 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:39,607 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/27fda8e44ca9484882be819ac4d268e2 2024-12-05T12:13:39,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/093ed70065084026b6d5ced4756b820a is 50, key is test_row_0/B:col10/1733400818170/Put/seqid=0 2024-12-05T12:13:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T12:13:39,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741925_1101 (size=12301) 2024-12-05T12:13:39,680 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/093ed70065084026b6d5ced4756b820a 2024-12-05T12:13:39,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400879683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:39,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:39,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400879684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:39,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400879684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:39,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/4ae16e9edde04a87bc9710cbc35529bb is 50, key is test_row_0/C:col10/1733400818170/Put/seqid=0 2024-12-05T12:13:39,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741926_1102 (size=12301) 2024-12-05T12:13:39,754 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/4ae16e9edde04a87bc9710cbc35529bb 2024-12-05T12:13:39,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/27fda8e44ca9484882be819ac4d268e2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/27fda8e44ca9484882be819ac4d268e2 2024-12-05T12:13:39,770 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/27fda8e44ca9484882be819ac4d268e2, entries=150, sequenceid=497, filesize=12.0 K 2024-12-05T12:13:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/093ed70065084026b6d5ced4756b820a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/093ed70065084026b6d5ced4756b820a 2024-12-05T12:13:39,784 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/093ed70065084026b6d5ced4756b820a, entries=150, sequenceid=497, filesize=12.0 K 2024-12-05T12:13:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/4ae16e9edde04a87bc9710cbc35529bb as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4ae16e9edde04a87bc9710cbc35529bb 2024-12-05T12:13:39,796 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4ae16e9edde04a87bc9710cbc35529bb, entries=150, sequenceid=497, filesize=12.0 K 2024-12-05T12:13:39,798 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 9c2bbad557c305bbacb16e1dfd91bc9a in 611ms, sequenceid=497, compaction requested=false 2024-12-05T12:13:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-05T12:13:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-05T12:13:39,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-05T12:13:39,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 770 msec 2024-12-05T12:13:39,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 793 msec 2024-12-05T12:13:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:40,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T12:13:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:40,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/36fa34febac84a639ab00f8296144ee5 is 50, key is test_row_0/A:col10/1733400820002/Put/seqid=0 2024-12-05T12:13:40,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400880058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741927_1103 (size=14741) 2024-12-05T12:13:40,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400880064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400880066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T12:13:40,175 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:40,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400880173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-05T12:13:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T12:13:40,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400880184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,187 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:40,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400880184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,191 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:40,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T12:13:40,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T12:13:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:40,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400880383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400880393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400880393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/36fa34febac84a639ab00f8296144ee5 2024-12-05T12:13:40,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/504251df68aa423cb6eda8b7ecbeec50 is 50, key is test_row_0/B:col10/1733400820002/Put/seqid=0 2024-12-05T12:13:40,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741928_1104 (size=12301) 2024-12-05T12:13:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T12:13:40,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T12:13:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:40,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T12:13:40,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:40,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:40,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:40,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400880692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400880703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400880703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:40,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T12:13:40,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:40,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:40,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:40,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T12:13:40,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/504251df68aa423cb6eda8b7ecbeec50 2024-12-05T12:13:40,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ec41f2bc5f0343ba9e72dfc5cebc455b is 50, key is test_row_0/C:col10/1733400820002/Put/seqid=0 2024-12-05T12:13:40,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741929_1105 (size=12301) 2024-12-05T12:13:40,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ec41f2bc5f0343ba9e72dfc5cebc455b 2024-12-05T12:13:40,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/36fa34febac84a639ab00f8296144ee5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/36fa34febac84a639ab00f8296144ee5 2024-12-05T12:13:40,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/36fa34febac84a639ab00f8296144ee5, entries=200, sequenceid=512, filesize=14.4 K 2024-12-05T12:13:40,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/504251df68aa423cb6eda8b7ecbeec50 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/504251df68aa423cb6eda8b7ecbeec50 2024-12-05T12:13:40,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T12:13:40,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:40,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:40,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:40,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:40,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/504251df68aa423cb6eda8b7ecbeec50, entries=150, sequenceid=512, filesize=12.0 K 2024-12-05T12:13:40,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/ec41f2bc5f0343ba9e72dfc5cebc455b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ec41f2bc5f0343ba9e72dfc5cebc455b 2024-12-05T12:13:40,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ec41f2bc5f0343ba9e72dfc5cebc455b, entries=150, sequenceid=512, filesize=12.0 K 2024-12-05T12:13:40,999 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-05T12:13:41,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9c2bbad557c305bbacb16e1dfd91bc9a in 997ms, sequenceid=512, compaction requested=true 2024-12-05T12:13:41,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:41,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:41,000 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:41,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:41,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:41,000 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:41,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:41,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:41,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:41,001 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:41,001 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:41,002 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:41,002 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:41,002 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/924785558ea14fc4b5e02796385a65fb, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/093ed70065084026b6d5ced4756b820a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/504251df68aa423cb6eda8b7ecbeec50] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=37.0 K 2024-12-05T12:13:41,002 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 924785558ea14fc4b5e02796385a65fb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733400818020 2024-12-05T12:13:41,002 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:41,002 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:41,003 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9918098fc3e740828b9322fb626c23e0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/27fda8e44ca9484882be819ac4d268e2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/36fa34febac84a639ab00f8296144ee5] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=39.4 K 2024-12-05T12:13:41,003 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 093ed70065084026b6d5ced4756b820a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733400818150 2024-12-05T12:13:41,003 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9918098fc3e740828b9322fb626c23e0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733400818020 2024-12-05T12:13:41,003 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 504251df68aa423cb6eda8b7ecbeec50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733400819996 2024-12-05T12:13:41,004 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27fda8e44ca9484882be819ac4d268e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733400818150 2024-12-05T12:13:41,005 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36fa34febac84a639ab00f8296144ee5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733400819358 2024-12-05T12:13:41,015 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#93 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:41,017 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/3b0a79f6c51d4554a931123e33eadebf is 50, key is test_row_0/B:col10/1733400820002/Put/seqid=0 2024-12-05T12:13:41,019 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:41,020 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/790fbee692fe4c018091a5a72937b233 is 50, key is test_row_0/A:col10/1733400820002/Put/seqid=0 2024-12-05T12:13:41,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741930_1106 (size=13357) 2024-12-05T12:13:41,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741931_1107 (size=13357) 2024-12-05T12:13:41,064 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/3b0a79f6c51d4554a931123e33eadebf as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/3b0a79f6c51d4554a931123e33eadebf 2024-12-05T12:13:41,072 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into 3b0a79f6c51d4554a931123e33eadebf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:41,072 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:41,072 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=13, startTime=1733400821000; duration=0sec 2024-12-05T12:13:41,073 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:41,073 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:41,074 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:41,077 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:41,077 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:41,077 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:41,078 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/8b1d214576e74d779593cf4598541063, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4ae16e9edde04a87bc9710cbc35529bb, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ec41f2bc5f0343ba9e72dfc5cebc455b] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=37.0 K 2024-12-05T12:13:41,078 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b1d214576e74d779593cf4598541063, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733400818020 2024-12-05T12:13:41,079 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ae16e9edde04a87bc9710cbc35529bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733400818150 2024-12-05T12:13:41,081 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting ec41f2bc5f0343ba9e72dfc5cebc455b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733400819996 2024-12-05T12:13:41,095 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:41,096 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/2c35d1c84a1e477abe32697345a60229 is 50, key is test_row_0/C:col10/1733400820002/Put/seqid=0 2024-12-05T12:13:41,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741932_1108 (size=13357) 2024-12-05T12:13:41,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T12:13:41,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:41,120 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T12:13:41,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:41,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:41,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:41,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:41,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:41,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:41,121 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/2c35d1c84a1e477abe32697345a60229 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/2c35d1c84a1e477abe32697345a60229 2024-12-05T12:13:41,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/e1017648aeae4225993ef5dedb654ec2 is 50, key is test_row_0/A:col10/1733400820062/Put/seqid=0 2024-12-05T12:13:41,134 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into 2c35d1c84a1e477abe32697345a60229(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:41,135 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:41,135 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=13, startTime=1733400821000; duration=0sec 2024-12-05T12:13:41,136 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:41,136 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:41,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741933_1109 (size=12301) 2024-12-05T12:13:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:41,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:41,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400881227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400881229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400881230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T12:13:41,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400881343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400881343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400881343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,447 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/790fbee692fe4c018091a5a72937b233 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/790fbee692fe4c018091a5a72937b233 2024-12-05T12:13:41,458 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into 790fbee692fe4c018091a5a72937b233(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:41,458 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:41,458 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=13, startTime=1733400821000; duration=0sec 2024-12-05T12:13:41,458 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:41,458 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:41,546 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/e1017648aeae4225993ef5dedb654ec2 2024-12-05T12:13:41,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400881553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400881553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400881553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/58ee594ab67843f7b5935ae0448b4d89 is 50, key is test_row_0/B:col10/1733400820062/Put/seqid=0 2024-12-05T12:13:41,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741934_1110 (size=12301) 2024-12-05T12:13:41,592 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/58ee594ab67843f7b5935ae0448b4d89 2024-12-05T12:13:41,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/d910a37e9be14b2db221e1540c5d473f is 50, key is test_row_0/C:col10/1733400820062/Put/seqid=0 2024-12-05T12:13:41,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741935_1111 (size=12301) 2024-12-05T12:13:41,638 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/d910a37e9be14b2db221e1540c5d473f 2024-12-05T12:13:41,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/e1017648aeae4225993ef5dedb654ec2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/e1017648aeae4225993ef5dedb654ec2 2024-12-05T12:13:41,658 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/e1017648aeae4225993ef5dedb654ec2, entries=150, sequenceid=531, filesize=12.0 K 2024-12-05T12:13:41,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/58ee594ab67843f7b5935ae0448b4d89 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/58ee594ab67843f7b5935ae0448b4d89 2024-12-05T12:13:41,671 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/58ee594ab67843f7b5935ae0448b4d89, entries=150, sequenceid=531, filesize=12.0 K 2024-12-05T12:13:41,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/d910a37e9be14b2db221e1540c5d473f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/d910a37e9be14b2db221e1540c5d473f 2024-12-05T12:13:41,687 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/d910a37e9be14b2db221e1540c5d473f, entries=150, sequenceid=531, filesize=12.0 K 2024-12-05T12:13:41,689 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9c2bbad557c305bbacb16e1dfd91bc9a in 570ms, sequenceid=531, compaction requested=false 2024-12-05T12:13:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-05T12:13:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-05T12:13:41,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-05T12:13:41,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5050 sec 2024-12-05T12:13:41,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.5230 sec 2024-12-05T12:13:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:41,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T12:13:41,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:41,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:41,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:41,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/14adcdb9fdc54e4c91614100074c4228 is 50, key is test_row_0/A:col10/1733400821863/Put/seqid=0 2024-12-05T12:13:41,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741936_1112 (size=12301) 2024-12-05T12:13:41,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/14adcdb9fdc54e4c91614100074c4228 2024-12-05T12:13:41,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400881887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400881887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400881889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/c54a598cc1b9470a959077ff99694e6d is 50, key is test_row_0/B:col10/1733400821863/Put/seqid=0 2024-12-05T12:13:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1733400881913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:41,917 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18307 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:41,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741937_1113 (size=12301) 2024-12-05T12:13:41,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/c54a598cc1b9470a959077ff99694e6d 2024-12-05T12:13:41,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/1b2eae7e29d84db195702d74db418b7b is 50, key is test_row_0/C:col10/1733400821863/Put/seqid=0 2024-12-05T12:13:41,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741938_1114 (size=12301) 2024-12-05T12:13:41,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/1b2eae7e29d84db195702d74db418b7b 2024-12-05T12:13:41,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/14adcdb9fdc54e4c91614100074c4228 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/14adcdb9fdc54e4c91614100074c4228 2024-12-05T12:13:41,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/14adcdb9fdc54e4c91614100074c4228, entries=150, sequenceid=550, filesize=12.0 K 2024-12-05T12:13:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/c54a598cc1b9470a959077ff99694e6d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c54a598cc1b9470a959077ff99694e6d 2024-12-05T12:13:42,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c54a598cc1b9470a959077ff99694e6d, entries=150, sequenceid=550, filesize=12.0 K 2024-12-05T12:13:42,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/1b2eae7e29d84db195702d74db418b7b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/1b2eae7e29d84db195702d74db418b7b 2024-12-05T12:13:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400882003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400882003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400882003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/1b2eae7e29d84db195702d74db418b7b, entries=150, sequenceid=550, filesize=12.0 K 2024-12-05T12:13:42,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9c2bbad557c305bbacb16e1dfd91bc9a in 151ms, sequenceid=550, compaction requested=true 2024-12-05T12:13:42,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:42,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:13:42,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:42,016 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:42,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:13:42,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:42,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c2bbad557c305bbacb16e1dfd91bc9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:13:42,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T12:13:42,017 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:42,018 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:42,018 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:42,018 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/A is initiating minor compaction (all files) 2024-12-05T12:13:42,018 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/B is initiating minor compaction (all files) 2024-12-05T12:13:42,018 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/A in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,018 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/B in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,018 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/790fbee692fe4c018091a5a72937b233, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/e1017648aeae4225993ef5dedb654ec2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/14adcdb9fdc54e4c91614100074c4228] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=37.1 K 2024-12-05T12:13:42,018 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/3b0a79f6c51d4554a931123e33eadebf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/58ee594ab67843f7b5935ae0448b4d89, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c54a598cc1b9470a959077ff99694e6d] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=37.1 K 2024-12-05T12:13:42,019 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b0a79f6c51d4554a931123e33eadebf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733400819996 2024-12-05T12:13:42,019 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 790fbee692fe4c018091a5a72937b233, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733400819996 2024-12-05T12:13:42,019 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1017648aeae4225993ef5dedb654ec2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733400820053 2024-12-05T12:13:42,019 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 58ee594ab67843f7b5935ae0448b4d89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733400820053 2024-12-05T12:13:42,020 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14adcdb9fdc54e4c91614100074c4228, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1733400821227 2024-12-05T12:13:42,020 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting c54a598cc1b9470a959077ff99694e6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1733400821227 2024-12-05T12:13:42,048 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:42,049 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/7fdd747b2b044ae9ad16bcb16b3fb073 is 50, key is test_row_0/A:col10/1733400821863/Put/seqid=0 2024-12-05T12:13:42,059 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:42,060 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/551708fd26844b21bc2abf8c69788adc is 50, key is test_row_0/B:col10/1733400821863/Put/seqid=0 2024-12-05T12:13:42,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741939_1115 (size=13459) 2024-12-05T12:13:42,114 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/7fdd747b2b044ae9ad16bcb16b3fb073 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/7fdd747b2b044ae9ad16bcb16b3fb073 2024-12-05T12:13:42,129 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/A of 9c2bbad557c305bbacb16e1dfd91bc9a into 7fdd747b2b044ae9ad16bcb16b3fb073(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:42,129 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:42,129 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/A, priority=13, startTime=1733400822016; duration=0sec 2024-12-05T12:13:42,129 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:13:42,129 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:A 2024-12-05T12:13:42,129 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:13:42,133 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:13:42,133 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 9c2bbad557c305bbacb16e1dfd91bc9a/C is initiating minor compaction (all files) 2024-12-05T12:13:42,133 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c2bbad557c305bbacb16e1dfd91bc9a/C in TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,134 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/2c35d1c84a1e477abe32697345a60229, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/d910a37e9be14b2db221e1540c5d473f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/1b2eae7e29d84db195702d74db418b7b] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp, totalSize=37.1 K 2024-12-05T12:13:42,135 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c35d1c84a1e477abe32697345a60229, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733400819996 2024-12-05T12:13:42,135 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting d910a37e9be14b2db221e1540c5d473f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733400820053 2024-12-05T12:13:42,136 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b2eae7e29d84db195702d74db418b7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1733400821227 2024-12-05T12:13:42,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741940_1116 (size=13459) 2024-12-05T12:13:42,150 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/551708fd26844b21bc2abf8c69788adc as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/551708fd26844b21bc2abf8c69788adc 2024-12-05T12:13:42,159 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/B of 9c2bbad557c305bbacb16e1dfd91bc9a into 551708fd26844b21bc2abf8c69788adc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:42,159 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:42,159 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/B, priority=13, startTime=1733400822016; duration=0sec 2024-12-05T12:13:42,159 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:42,159 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:B 2024-12-05T12:13:42,165 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c2bbad557c305bbacb16e1dfd91bc9a#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:13:42,166 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/aa9c6e91d2704f68bb2f2fb60d6e0c36 is 50, key is test_row_0/C:col10/1733400821863/Put/seqid=0 2024-12-05T12:13:42,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741941_1117 (size=13459) 2024-12-05T12:13:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:42,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T12:13:42,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:42,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:42,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:42,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:42,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:42,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:42,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/11b7eb0ac15945a5a7afc0afe4c23b7e is 50, key is test_row_0/A:col10/1733400822228/Put/seqid=0 2024-12-05T12:13:42,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400882301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400882301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400882303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T12:13:42,334 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:42,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-05T12:13:42,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741942_1118 (size=12301) 2024-12-05T12:13:42,338 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T12:13:42,343 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:42,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:42,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400882423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400882423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400882424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T12:13:42,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T12:13:42,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:42,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,605 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/aa9c6e91d2704f68bb2f2fb60d6e0c36 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/aa9c6e91d2704f68bb2f2fb60d6e0c36 2024-12-05T12:13:42,612 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c2bbad557c305bbacb16e1dfd91bc9a/C of 9c2bbad557c305bbacb16e1dfd91bc9a into aa9c6e91d2704f68bb2f2fb60d6e0c36(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:13:42,612 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:42,612 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., storeName=9c2bbad557c305bbacb16e1dfd91bc9a/C, priority=13, startTime=1733400822016; duration=0sec 2024-12-05T12:13:42,612 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:13:42,612 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c2bbad557c305bbacb16e1dfd91bc9a:C 2024-12-05T12:13:42,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400882633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400882633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400882633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T12:13:42,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:42,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T12:13:42,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/11b7eb0ac15945a5a7afc0afe4c23b7e 2024-12-05T12:13:42,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/77e9903a164240cc85658684142be571 is 50, key is test_row_0/B:col10/1733400822228/Put/seqid=0 2024-12-05T12:13:42,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741943_1119 (size=12301) 2024-12-05T12:13:42,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T12:13:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:42,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400882943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400882943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400882943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:42,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T12:13:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:42,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T12:13:43,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T12:13:43,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:43,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:43,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:43,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/77e9903a164240cc85658684142be571 2024-12-05T12:13:43,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/7f9fa66378e243f7939559b5e3fd69e4 is 50, key is test_row_0/C:col10/1733400822228/Put/seqid=0 2024-12-05T12:13:43,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741944_1120 (size=12301) 2024-12-05T12:13:43,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T12:13:43,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:43,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:43,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:43,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,295 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-05T12:13:43,295 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:13:43,296 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-05T12:13:43,296 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:13:43,298 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-05T12:13:43,298 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:13:43,299 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:43,299 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:43,299 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:43,299 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:43,299 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:43,299 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:43,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:13:43,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:13:43,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:13:43,300 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-05T12:13:43,300 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:13:43,300 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:43,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:13:43,300 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:43,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T12:13:43,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:43,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:43,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:43,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:43,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43054 deadline: 1733400883453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43008 deadline: 1733400883453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733400883454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T12:13:43,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43036 deadline: 1733400883523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:43,525 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in 9c2bbad557c305bbacb16e1dfd91bc9a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:13:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:13:43,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T12:13:43,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:43,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:43,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:43,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:43,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/7f9fa66378e243f7939559b5e3fd69e4 2024-12-05T12:13:43,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/11b7eb0ac15945a5a7afc0afe4c23b7e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/11b7eb0ac15945a5a7afc0afe4c23b7e 2024-12-05T12:13:43,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/11b7eb0ac15945a5a7afc0afe4c23b7e, entries=150, sequenceid=570, filesize=12.0 K 2024-12-05T12:13:43,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/77e9903a164240cc85658684142be571 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/77e9903a164240cc85658684142be571 2024-12-05T12:13:43,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/77e9903a164240cc85658684142be571, entries=150, sequenceid=570, filesize=12.0 K 2024-12-05T12:13:43,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/7f9fa66378e243f7939559b5e3fd69e4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/7f9fa66378e243f7939559b5e3fd69e4 2024-12-05T12:13:43,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/7f9fa66378e243f7939559b5e3fd69e4, entries=150, sequenceid=570, filesize=12.0 K 2024-12-05T12:13:43,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9c2bbad557c305bbacb16e1dfd91bc9a in 1354ms, sequenceid=570, compaction requested=false 2024-12-05T12:13:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:43,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T12:13:43,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:43,736 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T12:13:43,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:43,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:43,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:43,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/7ae2f11b143749f78a2d8169f7fe89ad is 50, key is test_row_0/A:col10/1733400822296/Put/seqid=0 2024-12-05T12:13:43,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741945_1121 (size=12301) 2024-12-05T12:13:44,153 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/7ae2f11b143749f78a2d8169f7fe89ad 2024-12-05T12:13:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7cae32a3d36244b28143d562233347cb is 50, key is test_row_0/B:col10/1733400822296/Put/seqid=0 2024-12-05T12:13:44,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741946_1122 (size=12301) 2024-12-05T12:13:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:44,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. as already flushing 2024-12-05T12:13:44,465 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-05T12:13:44,465 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-05T12:13:44,465 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:13:44,465 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:13:44,465 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:44,465 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:44,465 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-05T12:13:44,465 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:13:44,465 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:44,465 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:44,465 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:44,465 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:44,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:13:44,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:13:44,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:13:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T12:13:44,515 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T12:13:44,569 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7cae32a3d36244b28143d562233347cb 2024-12-05T12:13:44,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/f806355a8486494398735ea3f9933dba is 50, key is test_row_0/C:col10/1733400822296/Put/seqid=0 2024-12-05T12:13:44,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741947_1123 (size=12301) 2024-12-05T12:13:44,984 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/f806355a8486494398735ea3f9933dba 2024-12-05T12:13:44,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/7ae2f11b143749f78a2d8169f7fe89ad as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/7ae2f11b143749f78a2d8169f7fe89ad 2024-12-05T12:13:44,993 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/7ae2f11b143749f78a2d8169f7fe89ad, entries=150, sequenceid=585, filesize=12.0 K 2024-12-05T12:13:44,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/7cae32a3d36244b28143d562233347cb as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7cae32a3d36244b28143d562233347cb 2024-12-05T12:13:44,999 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7cae32a3d36244b28143d562233347cb, entries=150, sequenceid=585, filesize=12.0 K 2024-12-05T12:13:45,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/f806355a8486494398735ea3f9933dba as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/f806355a8486494398735ea3f9933dba 2024-12-05T12:13:45,005 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/f806355a8486494398735ea3f9933dba, entries=150, sequenceid=585, filesize=12.0 K 2024-12-05T12:13:45,006 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for 9c2bbad557c305bbacb16e1dfd91bc9a in 1270ms, sequenceid=585, compaction requested=true 2024-12-05T12:13:45,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 9c2bbad557c305bbacb16e1dfd91bc9a: 2024-12-05T12:13:45,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:45,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-05T12:13:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-05T12:13:45,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-05T12:13:45,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6640 sec 2024-12-05T12:13:45,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 2.6740 sec 2024-12-05T12:13:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T12:13:46,504 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:51,955 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-05T12:13:51,955 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:13:51,955 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:51,955 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:51,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:13:53,559 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-05T12:13:53,559 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:13:53,559 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:53,560 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:53,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:13:53,560 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T12:13:53,560 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-12-05T12:13:53,560 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-12-05T12:13:53,560 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-12-05T12:13:53,560 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-05T12:13:53,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-12-05T12:13:53,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T12:13:53,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3189 2024-12-05T12:13:53,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3147 2024-12-05T12:13:53,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T12:13:53,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1461 2024-12-05T12:13:53,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4376 rows 2024-12-05T12:13:53,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1460 2024-12-05T12:13:53,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4377 rows 2024-12-05T12:13:53,562 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:13:53,562 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:13:53,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:53,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:53,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:13:53,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T12:13:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T12:13:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T12:13:53,576 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400833575"}]},"ts":"1733400833575"} 2024-12-05T12:13:53,577 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T12:13:53,578 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T12:13:53,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T12:13:53,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c2bbad557c305bbacb16e1dfd91bc9a, UNASSIGN}] 2024-12-05T12:13:53,584 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c2bbad557c305bbacb16e1dfd91bc9a, UNASSIGN 2024-12-05T12:13:53,585 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=9c2bbad557c305bbacb16e1dfd91bc9a, regionState=CLOSING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:53,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c2bbad557c305bbacb16e1dfd91bc9a, UNASSIGN because future has completed 2024-12-05T12:13:53,588 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T12:13:53,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:13:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T12:13:53,749 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:53,749 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T12:13:53,751 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 9c2bbad557c305bbacb16e1dfd91bc9a, disabling compactions & flushes 2024-12-05T12:13:53,751 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:53,752 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:53,752 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. after waiting 0 ms 2024-12-05T12:13:53,752 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:53,752 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 9c2bbad557c305bbacb16e1dfd91bc9a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T12:13:53,753 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=A 2024-12-05T12:13:53,753 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:53,753 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=B 2024-12-05T12:13:53,753 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:53,753 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c2bbad557c305bbacb16e1dfd91bc9a, store=C 2024-12-05T12:13:53,753 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:53,760 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/e7f4e54ac7654f9f9152d82149c15743 is 50, key is test_row_0/A:col10/1733400831953/Put/seqid=0 2024-12-05T12:13:53,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741948_1124 (size=12301) 2024-12-05T12:13:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T12:13:54,166 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/e7f4e54ac7654f9f9152d82149c15743 2024-12-05T12:13:54,179 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/5189395566e744ec8b48d20edc06ea8a is 50, key is test_row_0/B:col10/1733400831953/Put/seqid=0 2024-12-05T12:13:54,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741949_1125 (size=12301) 2024-12-05T12:13:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T12:13:54,585 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/5189395566e744ec8b48d20edc06ea8a 2024-12-05T12:13:54,598 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/79c6068ef7bb4c73ad298cc05f9a9713 is 50, key is test_row_0/C:col10/1733400831953/Put/seqid=0 2024-12-05T12:13:54,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741950_1126 (size=12301) 2024-12-05T12:13:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T12:13:55,004 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/79c6068ef7bb4c73ad298cc05f9a9713 2024-12-05T12:13:55,009 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/A/e7f4e54ac7654f9f9152d82149c15743 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/e7f4e54ac7654f9f9152d82149c15743 2024-12-05T12:13:55,014 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/e7f4e54ac7654f9f9152d82149c15743, entries=150, sequenceid=593, filesize=12.0 K 2024-12-05T12:13:55,015 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/B/5189395566e744ec8b48d20edc06ea8a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/5189395566e744ec8b48d20edc06ea8a 2024-12-05T12:13:55,019 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/5189395566e744ec8b48d20edc06ea8a, entries=150, sequenceid=593, filesize=12.0 K 2024-12-05T12:13:55,020 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/.tmp/C/79c6068ef7bb4c73ad298cc05f9a9713 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/79c6068ef7bb4c73ad298cc05f9a9713 2024-12-05T12:13:55,025 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/79c6068ef7bb4c73ad298cc05f9a9713, entries=150, sequenceid=593, filesize=12.0 K 2024-12-05T12:13:55,026 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9c2bbad557c305bbacb16e1dfd91bc9a in 1274ms, sequenceid=593, compaction requested=true 2024-12-05T12:13:55,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a3f4192ab9f3406a87bcfa98517e7e43, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/d82f368c0d03411791aeba52aa27e72f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/3ee859e669f84e7c9b78c9732405300e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/70775de9ac854fe481ee4d0693cc92fd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/457263c9db4544598136a46520b8117e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/ea043dfb4068485b976605709f4eb838, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/103b3fac3b224bb4a12ffa25ffeb56df, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/86e6d71c83cf4f25bb7ea2e5f0b9e022, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/5989c33d27ad4808bfa509c2951ee0bd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/f9aa4514356c475fa9d75bc0d6f5876f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b1f771f712b4436da3981fd356d04fa7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/eec0c0b0e08a4f4fa4f077c09ebd9394, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c0e6fc1e44424c8e9c0b604e18378101, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/80d9211931c64f5db96dab99dbac5395, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6afbe5efcfc042e2b85e2c458a18611a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c6efd996dde64477b8fc0772d9714960, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6f74d5d18fb04426ab5b022e0656b149, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/823b587f72d54d2dab129d119af44038, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9ec19e68c86c45c3bd3b5df396837e38, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/30693d5314134472a6100136dcc3c42b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/055601fa9a2f48e991c969ea1a6bf34a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/cb9d38c57a92415dbe30e6f991e66037, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a8ce92768d3b498e9a37c25a6a4f56bd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b99ec3abb56a469eadbab6a5120b5477, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/23073b766d0f490cae82cecdf1baba17, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c8a788b5300244f7b26105dfdbfcf451, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b6723cb1b1f7479295bdf78b74ec4909, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a2f2e3650cdf4bcc8660cfe087f2433f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9918098fc3e740828b9322fb626c23e0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/27fda8e44ca9484882be819ac4d268e2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/36fa34febac84a639ab00f8296144ee5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/790fbee692fe4c018091a5a72937b233, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/e1017648aeae4225993ef5dedb654ec2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/14adcdb9fdc54e4c91614100074c4228] to archive 2024-12-05T12:13:55,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:13:55,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a3f4192ab9f3406a87bcfa98517e7e43 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a3f4192ab9f3406a87bcfa98517e7e43 2024-12-05T12:13:55,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/d82f368c0d03411791aeba52aa27e72f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/d82f368c0d03411791aeba52aa27e72f 2024-12-05T12:13:55,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/3ee859e669f84e7c9b78c9732405300e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/3ee859e669f84e7c9b78c9732405300e 2024-12-05T12:13:55,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/70775de9ac854fe481ee4d0693cc92fd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/70775de9ac854fe481ee4d0693cc92fd 2024-12-05T12:13:55,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/457263c9db4544598136a46520b8117e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/457263c9db4544598136a46520b8117e 2024-12-05T12:13:55,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/ea043dfb4068485b976605709f4eb838 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/ea043dfb4068485b976605709f4eb838 2024-12-05T12:13:55,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/103b3fac3b224bb4a12ffa25ffeb56df to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/103b3fac3b224bb4a12ffa25ffeb56df 2024-12-05T12:13:55,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/86e6d71c83cf4f25bb7ea2e5f0b9e022 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/86e6d71c83cf4f25bb7ea2e5f0b9e022 2024-12-05T12:13:55,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/5989c33d27ad4808bfa509c2951ee0bd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/5989c33d27ad4808bfa509c2951ee0bd 2024-12-05T12:13:55,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/f9aa4514356c475fa9d75bc0d6f5876f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/f9aa4514356c475fa9d75bc0d6f5876f 2024-12-05T12:13:55,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b1f771f712b4436da3981fd356d04fa7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b1f771f712b4436da3981fd356d04fa7 2024-12-05T12:13:55,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/eec0c0b0e08a4f4fa4f077c09ebd9394 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/eec0c0b0e08a4f4fa4f077c09ebd9394 2024-12-05T12:13:55,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c0e6fc1e44424c8e9c0b604e18378101 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c0e6fc1e44424c8e9c0b604e18378101 2024-12-05T12:13:55,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/80d9211931c64f5db96dab99dbac5395 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/80d9211931c64f5db96dab99dbac5395 2024-12-05T12:13:55,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6afbe5efcfc042e2b85e2c458a18611a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6afbe5efcfc042e2b85e2c458a18611a 2024-12-05T12:13:55,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c6efd996dde64477b8fc0772d9714960 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c6efd996dde64477b8fc0772d9714960 2024-12-05T12:13:55,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6f74d5d18fb04426ab5b022e0656b149 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/6f74d5d18fb04426ab5b022e0656b149 2024-12-05T12:13:55,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/823b587f72d54d2dab129d119af44038 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/823b587f72d54d2dab129d119af44038 2024-12-05T12:13:55,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9ec19e68c86c45c3bd3b5df396837e38 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9ec19e68c86c45c3bd3b5df396837e38 2024-12-05T12:13:55,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/30693d5314134472a6100136dcc3c42b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/30693d5314134472a6100136dcc3c42b 2024-12-05T12:13:55,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/055601fa9a2f48e991c969ea1a6bf34a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/055601fa9a2f48e991c969ea1a6bf34a 2024-12-05T12:13:55,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/cb9d38c57a92415dbe30e6f991e66037 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/cb9d38c57a92415dbe30e6f991e66037 2024-12-05T12:13:55,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a8ce92768d3b498e9a37c25a6a4f56bd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a8ce92768d3b498e9a37c25a6a4f56bd 2024-12-05T12:13:55,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b99ec3abb56a469eadbab6a5120b5477 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b99ec3abb56a469eadbab6a5120b5477 2024-12-05T12:13:55,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/23073b766d0f490cae82cecdf1baba17 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/23073b766d0f490cae82cecdf1baba17 2024-12-05T12:13:55,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c8a788b5300244f7b26105dfdbfcf451 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/c8a788b5300244f7b26105dfdbfcf451 2024-12-05T12:13:55,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b6723cb1b1f7479295bdf78b74ec4909 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/b6723cb1b1f7479295bdf78b74ec4909 2024-12-05T12:13:55,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a2f2e3650cdf4bcc8660cfe087f2433f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/a2f2e3650cdf4bcc8660cfe087f2433f 2024-12-05T12:13:55,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9918098fc3e740828b9322fb626c23e0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/9918098fc3e740828b9322fb626c23e0 2024-12-05T12:13:55,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/27fda8e44ca9484882be819ac4d268e2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/27fda8e44ca9484882be819ac4d268e2 2024-12-05T12:13:55,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/36fa34febac84a639ab00f8296144ee5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/36fa34febac84a639ab00f8296144ee5 2024-12-05T12:13:55,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/790fbee692fe4c018091a5a72937b233 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/790fbee692fe4c018091a5a72937b233 2024-12-05T12:13:55,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/e1017648aeae4225993ef5dedb654ec2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/e1017648aeae4225993ef5dedb654ec2 2024-12-05T12:13:55,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/14adcdb9fdc54e4c91614100074c4228 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/14adcdb9fdc54e4c91614100074c4228 2024-12-05T12:13:55,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/5c793a368fe54b04996a36958d3d0b86, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/6bb0734edc0040209a220618322f7388, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f06ac642462e43ad94013bb514bb03f1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/b2ae764790224f1f8f758b79487b455c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4072a8254d9a46d6bcb4dde4c8e3b58d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/67722cd9e398491ca3ad73735864dc66, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/2e5c5863806643bcbef4decec42d9fa7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/efce2637dbf54edaaf03274f707d071e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7b65b896f2ae44c9b4036be750103980, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c108df6bb4e74315a6ca70ba99c0da7c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7663c02549e54c0696ba4ddff2d64fe4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f35c72c2710548de9f1c9a81916b6b37, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/bad0ce9f440141a3937d707585dc7c4c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4962cf232b604fe9b39849086eebb162, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/3bc060a1a3144999a7aac2d6140a4e89, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f585c0b3c1e2407f839c8dc46f5c3dd5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/177b7ab283404133a98916260ef51370, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/d86823476e8d4ef9b7321e62214ea896, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8debc5bd2540451690e94cbd1b2f2c5a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8e1bcfaf2da74cd2b883f604de5e41fd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/995cc6cfefd44a5aa06890f4101171ef, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/54ce1469c9be4409bdac0ec283bc3bed, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/0e22ff7d8d5c4d748d63966e4154da49, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/a40154a144694065b28dae9b070d86a2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8cead26d29cc49c794356de39ed16d51, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7990048fd36140ddae7862cb48f1ba77, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/38ec016ac21f4ef987d8cbcd70cce6b0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/924785558ea14fc4b5e02796385a65fb, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/9b70ff431bcf472b8a10949e5a8cbb69, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/093ed70065084026b6d5ced4756b820a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/3b0a79f6c51d4554a931123e33eadebf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/504251df68aa423cb6eda8b7ecbeec50, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/58ee594ab67843f7b5935ae0448b4d89, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c54a598cc1b9470a959077ff99694e6d] to archive 2024-12-05T12:13:55,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:13:55,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/5c793a368fe54b04996a36958d3d0b86 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/5c793a368fe54b04996a36958d3d0b86 2024-12-05T12:13:55,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/6bb0734edc0040209a220618322f7388 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/6bb0734edc0040209a220618322f7388 2024-12-05T12:13:55,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f06ac642462e43ad94013bb514bb03f1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f06ac642462e43ad94013bb514bb03f1 2024-12-05T12:13:55,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/b2ae764790224f1f8f758b79487b455c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/b2ae764790224f1f8f758b79487b455c 2024-12-05T12:13:55,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4072a8254d9a46d6bcb4dde4c8e3b58d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4072a8254d9a46d6bcb4dde4c8e3b58d 2024-12-05T12:13:55,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/67722cd9e398491ca3ad73735864dc66 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/67722cd9e398491ca3ad73735864dc66 2024-12-05T12:13:55,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/2e5c5863806643bcbef4decec42d9fa7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/2e5c5863806643bcbef4decec42d9fa7 2024-12-05T12:13:55,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/efce2637dbf54edaaf03274f707d071e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/efce2637dbf54edaaf03274f707d071e 2024-12-05T12:13:55,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7b65b896f2ae44c9b4036be750103980 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7b65b896f2ae44c9b4036be750103980 2024-12-05T12:13:55,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c108df6bb4e74315a6ca70ba99c0da7c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c108df6bb4e74315a6ca70ba99c0da7c 2024-12-05T12:13:55,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7663c02549e54c0696ba4ddff2d64fe4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7663c02549e54c0696ba4ddff2d64fe4 2024-12-05T12:13:55,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f35c72c2710548de9f1c9a81916b6b37 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f35c72c2710548de9f1c9a81916b6b37 2024-12-05T12:13:55,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/bad0ce9f440141a3937d707585dc7c4c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/bad0ce9f440141a3937d707585dc7c4c 2024-12-05T12:13:55,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4962cf232b604fe9b39849086eebb162 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/4962cf232b604fe9b39849086eebb162 2024-12-05T12:13:55,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/3bc060a1a3144999a7aac2d6140a4e89 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/3bc060a1a3144999a7aac2d6140a4e89 2024-12-05T12:13:55,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f585c0b3c1e2407f839c8dc46f5c3dd5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/f585c0b3c1e2407f839c8dc46f5c3dd5 2024-12-05T12:13:55,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/177b7ab283404133a98916260ef51370 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/177b7ab283404133a98916260ef51370 2024-12-05T12:13:55,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/d86823476e8d4ef9b7321e62214ea896 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/d86823476e8d4ef9b7321e62214ea896 2024-12-05T12:13:55,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8debc5bd2540451690e94cbd1b2f2c5a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8debc5bd2540451690e94cbd1b2f2c5a 2024-12-05T12:13:55,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8e1bcfaf2da74cd2b883f604de5e41fd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8e1bcfaf2da74cd2b883f604de5e41fd 2024-12-05T12:13:55,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/995cc6cfefd44a5aa06890f4101171ef to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/995cc6cfefd44a5aa06890f4101171ef 2024-12-05T12:13:55,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/54ce1469c9be4409bdac0ec283bc3bed to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/54ce1469c9be4409bdac0ec283bc3bed 2024-12-05T12:13:55,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/0e22ff7d8d5c4d748d63966e4154da49 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/0e22ff7d8d5c4d748d63966e4154da49 2024-12-05T12:13:55,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/a40154a144694065b28dae9b070d86a2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/a40154a144694065b28dae9b070d86a2 2024-12-05T12:13:55,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8cead26d29cc49c794356de39ed16d51 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/8cead26d29cc49c794356de39ed16d51 2024-12-05T12:13:55,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7990048fd36140ddae7862cb48f1ba77 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7990048fd36140ddae7862cb48f1ba77 2024-12-05T12:13:55,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/38ec016ac21f4ef987d8cbcd70cce6b0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/38ec016ac21f4ef987d8cbcd70cce6b0 2024-12-05T12:13:55,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/924785558ea14fc4b5e02796385a65fb to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/924785558ea14fc4b5e02796385a65fb 2024-12-05T12:13:55,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/9b70ff431bcf472b8a10949e5a8cbb69 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/9b70ff431bcf472b8a10949e5a8cbb69 2024-12-05T12:13:55,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/093ed70065084026b6d5ced4756b820a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/093ed70065084026b6d5ced4756b820a 2024-12-05T12:13:55,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/3b0a79f6c51d4554a931123e33eadebf to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/3b0a79f6c51d4554a931123e33eadebf 2024-12-05T12:13:55,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/504251df68aa423cb6eda8b7ecbeec50 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/504251df68aa423cb6eda8b7ecbeec50 2024-12-05T12:13:55,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/58ee594ab67843f7b5935ae0448b4d89 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/58ee594ab67843f7b5935ae0448b4d89 2024-12-05T12:13:55,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c54a598cc1b9470a959077ff99694e6d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/c54a598cc1b9470a959077ff99694e6d 2024-12-05T12:13:55,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/226669c6838b4353b8a05e7049fda68f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ee0e946ab5eb47beb689dd5c05db9638, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b3ca07680b3d40458b48a5760efbcd2a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/adfaf98dc484484081557d031a98e2ba, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b34454f575714ea09f97feec74515e20, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/24bcaa9df6b141a09f2792a59e66d3f9, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ba50508fcf4242f2bd0a10bfc4aa185d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c1fd03abac004c2193fc60efb1c30f64, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/46ead7304d774ca190a420a339bfd89a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c63b46f53d474e3eb5af43371b9bea0b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/5bc107a911fd4b418bc891af9200a434, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e9544160d02c4dcea8fe0091951dd71c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/3c04993f0e814b70bc5701aaa7973f6b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/a3d308c15f4743b9b5646fd09877fdaa, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/716a01a3005f47fc894d98e99291fddd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/faac1939b3a9483596091ddb09bfde94, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4cc1980a782641ce8d49dcea2078195c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/00d2c0ec4aeb4b55bb882e6aa7aa07b6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/76961b1fbe7b43b48a94789524ae1c76, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ad93db5180444ebe9251f992290030e4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/74551749a0d74f628a3bd711227b0c6a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/796b85db627e4c538648b901c461d3c7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/215b510517344e0582818be13fe140b3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/0ebc6dd2c20c4116ab3e659494929616, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/18ec9be7fd7043ea8acb8e85813035c3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e1c5e5415bd641178f962a5621392685, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/fba26d331aac44a8a6885725d952f674, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/8b1d214576e74d779593cf4598541063, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/837e0a20635646e8853f603296ec9b3a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4ae16e9edde04a87bc9710cbc35529bb, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/2c35d1c84a1e477abe32697345a60229, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ec41f2bc5f0343ba9e72dfc5cebc455b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/d910a37e9be14b2db221e1540c5d473f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/1b2eae7e29d84db195702d74db418b7b] to archive 2024-12-05T12:13:55,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:13:55,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/226669c6838b4353b8a05e7049fda68f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/226669c6838b4353b8a05e7049fda68f 2024-12-05T12:13:55,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ee0e946ab5eb47beb689dd5c05db9638 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ee0e946ab5eb47beb689dd5c05db9638 2024-12-05T12:13:55,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b3ca07680b3d40458b48a5760efbcd2a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b3ca07680b3d40458b48a5760efbcd2a 2024-12-05T12:13:55,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/adfaf98dc484484081557d031a98e2ba to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/adfaf98dc484484081557d031a98e2ba 2024-12-05T12:13:55,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b34454f575714ea09f97feec74515e20 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/b34454f575714ea09f97feec74515e20 2024-12-05T12:13:55,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/24bcaa9df6b141a09f2792a59e66d3f9 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/24bcaa9df6b141a09f2792a59e66d3f9 2024-12-05T12:13:55,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ba50508fcf4242f2bd0a10bfc4aa185d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ba50508fcf4242f2bd0a10bfc4aa185d 2024-12-05T12:13:55,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c1fd03abac004c2193fc60efb1c30f64 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c1fd03abac004c2193fc60efb1c30f64 2024-12-05T12:13:55,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/46ead7304d774ca190a420a339bfd89a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/46ead7304d774ca190a420a339bfd89a 2024-12-05T12:13:55,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c63b46f53d474e3eb5af43371b9bea0b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/c63b46f53d474e3eb5af43371b9bea0b 2024-12-05T12:13:55,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/5bc107a911fd4b418bc891af9200a434 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/5bc107a911fd4b418bc891af9200a434 2024-12-05T12:13:55,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e9544160d02c4dcea8fe0091951dd71c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e9544160d02c4dcea8fe0091951dd71c 2024-12-05T12:13:55,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/3c04993f0e814b70bc5701aaa7973f6b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/3c04993f0e814b70bc5701aaa7973f6b 2024-12-05T12:13:55,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/a3d308c15f4743b9b5646fd09877fdaa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/a3d308c15f4743b9b5646fd09877fdaa 2024-12-05T12:13:55,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/716a01a3005f47fc894d98e99291fddd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/716a01a3005f47fc894d98e99291fddd 2024-12-05T12:13:55,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/faac1939b3a9483596091ddb09bfde94 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/faac1939b3a9483596091ddb09bfde94 2024-12-05T12:13:55,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4cc1980a782641ce8d49dcea2078195c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4cc1980a782641ce8d49dcea2078195c 2024-12-05T12:13:55,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/00d2c0ec4aeb4b55bb882e6aa7aa07b6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/00d2c0ec4aeb4b55bb882e6aa7aa07b6 2024-12-05T12:13:55,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/76961b1fbe7b43b48a94789524ae1c76 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/76961b1fbe7b43b48a94789524ae1c76 2024-12-05T12:13:55,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ad93db5180444ebe9251f992290030e4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ad93db5180444ebe9251f992290030e4 2024-12-05T12:13:55,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/74551749a0d74f628a3bd711227b0c6a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/74551749a0d74f628a3bd711227b0c6a 2024-12-05T12:13:55,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/796b85db627e4c538648b901c461d3c7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/796b85db627e4c538648b901c461d3c7 2024-12-05T12:13:55,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/215b510517344e0582818be13fe140b3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/215b510517344e0582818be13fe140b3 2024-12-05T12:13:55,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/0ebc6dd2c20c4116ab3e659494929616 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/0ebc6dd2c20c4116ab3e659494929616 2024-12-05T12:13:55,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/18ec9be7fd7043ea8acb8e85813035c3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/18ec9be7fd7043ea8acb8e85813035c3 2024-12-05T12:13:55,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e1c5e5415bd641178f962a5621392685 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/e1c5e5415bd641178f962a5621392685 2024-12-05T12:13:55,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/fba26d331aac44a8a6885725d952f674 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/fba26d331aac44a8a6885725d952f674 2024-12-05T12:13:55,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/8b1d214576e74d779593cf4598541063 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/8b1d214576e74d779593cf4598541063 2024-12-05T12:13:55,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/837e0a20635646e8853f603296ec9b3a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/837e0a20635646e8853f603296ec9b3a 2024-12-05T12:13:55,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4ae16e9edde04a87bc9710cbc35529bb to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/4ae16e9edde04a87bc9710cbc35529bb 2024-12-05T12:13:55,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/2c35d1c84a1e477abe32697345a60229 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/2c35d1c84a1e477abe32697345a60229 2024-12-05T12:13:55,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ec41f2bc5f0343ba9e72dfc5cebc455b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/ec41f2bc5f0343ba9e72dfc5cebc455b 2024-12-05T12:13:55,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/d910a37e9be14b2db221e1540c5d473f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/d910a37e9be14b2db221e1540c5d473f 2024-12-05T12:13:55,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/1b2eae7e29d84db195702d74db418b7b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/1b2eae7e29d84db195702d74db418b7b 2024-12-05T12:13:55,202 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/recovered.edits/596.seqid, newMaxSeqId=596, maxSeqId=1 2024-12-05T12:13:55,204 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. 2024-12-05T12:13:55,204 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 9c2bbad557c305bbacb16e1dfd91bc9a: Waiting for close lock at 1733400833751Running coprocessor pre-close hooks at 1733400833751Disabling compacts and flushes for region at 1733400833751Disabling writes for close at 1733400833752 (+1 ms)Obtaining lock to block concurrent updates at 1733400833752Preparing flush snapshotting stores in 9c2bbad557c305bbacb16e1dfd91bc9a at 1733400833752Finished memstore snapshotting TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733400833753 (+1 ms)Flushing stores of TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a. at 1733400833754 (+1 ms)Flushing 9c2bbad557c305bbacb16e1dfd91bc9a/A: creating writer at 1733400833754Flushing 9c2bbad557c305bbacb16e1dfd91bc9a/A: appending metadata at 1733400833759 (+5 ms)Flushing 9c2bbad557c305bbacb16e1dfd91bc9a/A: closing flushed file at 1733400833759Flushing 9c2bbad557c305bbacb16e1dfd91bc9a/B: creating writer at 1733400834175 (+416 ms)Flushing 9c2bbad557c305bbacb16e1dfd91bc9a/B: appending metadata at 1733400834179 (+4 ms)Flushing 9c2bbad557c305bbacb16e1dfd91bc9a/B: closing flushed file at 1733400834179Flushing 9c2bbad557c305bbacb16e1dfd91bc9a/C: creating writer at 1733400834594 (+415 ms)Flushing 9c2bbad557c305bbacb16e1dfd91bc9a/C: appending metadata at 1733400834598 (+4 ms)Flushing 9c2bbad557c305bbacb16e1dfd91bc9a/C: closing flushed file at 1733400834598Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b2634b0: reopening flushed file at 1733400835009 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b30cfd4: reopening flushed file at 1733400835014 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59e6b897: reopening flushed file at 1733400835020 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9c2bbad557c305bbacb16e1dfd91bc9a in 1274ms, sequenceid=593, compaction requested=true at 1733400835026 (+6 ms)Writing region close event to WAL at 1733400835198 (+172 ms)Running coprocessor post-close hooks at 1733400835203 (+5 ms)Closed at 1733400835203 2024-12-05T12:13:55,207 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:55,208 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=9c2bbad557c305bbacb16e1dfd91bc9a, regionState=CLOSED 2024-12-05T12:13:55,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:13:55,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-05T12:13:55,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 9c2bbad557c305bbacb16e1dfd91bc9a, server=9ea9abad0e9b,44759,1733400796278 in 1.6230 sec 2024-12-05T12:13:55,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-05T12:13:55,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c2bbad557c305bbacb16e1dfd91bc9a, UNASSIGN in 1.6300 sec 2024-12-05T12:13:55,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-05T12:13:55,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6370 sec 2024-12-05T12:13:55,222 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400835222"}]},"ts":"1733400835222"} 2024-12-05T12:13:55,224 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T12:13:55,225 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T12:13:55,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6570 sec 2024-12-05T12:13:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T12:13:55,725 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:55,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T12:13:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:13:55,739 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:13:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-05T12:13:55,743 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:13:55,755 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:55,759 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/recovered.edits] 2024-12-05T12:13:55,762 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/11b7eb0ac15945a5a7afc0afe4c23b7e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/11b7eb0ac15945a5a7afc0afe4c23b7e 2024-12-05T12:13:55,764 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/7ae2f11b143749f78a2d8169f7fe89ad to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/7ae2f11b143749f78a2d8169f7fe89ad 2024-12-05T12:13:55,765 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/7fdd747b2b044ae9ad16bcb16b3fb073 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/7fdd747b2b044ae9ad16bcb16b3fb073 2024-12-05T12:13:55,767 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/e7f4e54ac7654f9f9152d82149c15743 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/A/e7f4e54ac7654f9f9152d82149c15743 2024-12-05T12:13:55,770 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/5189395566e744ec8b48d20edc06ea8a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/5189395566e744ec8b48d20edc06ea8a 2024-12-05T12:13:55,772 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/551708fd26844b21bc2abf8c69788adc to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/551708fd26844b21bc2abf8c69788adc 2024-12-05T12:13:55,775 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/77e9903a164240cc85658684142be571 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/77e9903a164240cc85658684142be571 2024-12-05T12:13:55,777 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7cae32a3d36244b28143d562233347cb to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/B/7cae32a3d36244b28143d562233347cb 2024-12-05T12:13:55,781 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/79c6068ef7bb4c73ad298cc05f9a9713 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/79c6068ef7bb4c73ad298cc05f9a9713 2024-12-05T12:13:55,782 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/7f9fa66378e243f7939559b5e3fd69e4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/7f9fa66378e243f7939559b5e3fd69e4 2024-12-05T12:13:55,784 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/aa9c6e91d2704f68bb2f2fb60d6e0c36 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/aa9c6e91d2704f68bb2f2fb60d6e0c36 2024-12-05T12:13:55,785 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/f806355a8486494398735ea3f9933dba to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/C/f806355a8486494398735ea3f9933dba 2024-12-05T12:13:55,789 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/recovered.edits/596.seqid to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a/recovered.edits/596.seqid 2024-12-05T12:13:55,790 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/9c2bbad557c305bbacb16e1dfd91bc9a 2024-12-05T12:13:55,790 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T12:13:55,796 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:13:55,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-05T12:13:55,811 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T12:13:55,849 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T12:13:55,851 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:13:55,851 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T12:13:55,852 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733400835851"}]},"ts":"9223372036854775807"} 2024-12-05T12:13:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-05T12:13:55,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T12:13:55,857 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9c2bbad557c305bbacb16e1dfd91bc9a, NAME => 'TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T12:13:55,857 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T12:13:55,857 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733400835857"}]},"ts":"9223372036854775807"} 2024-12-05T12:13:55,861 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T12:13:55,862 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:13:55,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 132 msec 2024-12-05T12:13:55,895 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T12:13:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-05T12:13:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T12:13:56,064 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:56,073 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9ea9abad0e9b:44759-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-144804023_22 at /127.0.0.1:42564 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=576 (was 444) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3641 (was 4496) 2024-12-05T12:13:56,082 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=576, ProcessCount=11, AvailableMemoryMB=3641 2024-12-05T12:13:56,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T12:13:56,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:13:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T12:13:56,087 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T12:13:56,087 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:56,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-12-05T12:13:56,088 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T12:13:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T12:13:56,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741951_1127 (size=1061) 2024-12-05T12:13:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T12:13:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T12:13:56,499 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e 2024-12-05T12:13:56,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741952_1128 (size=53) 2024-12-05T12:13:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T12:13:56,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:13:56,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dde686338ee027649f1546b042d1ca1b, disabling compactions & flushes 2024-12-05T12:13:56,908 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:56,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:56,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. after waiting 0 ms 2024-12-05T12:13:56,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:56,908 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:56,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dde686338ee027649f1546b042d1ca1b: Waiting for close lock at 1733400836908Disabling compacts and flushes for region at 1733400836908Disabling writes for close at 1733400836908Writing region close event to WAL at 1733400836908Closed at 1733400836908 2024-12-05T12:13:56,910 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T12:13:56,910 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733400836910"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733400836910"}]},"ts":"1733400836910"} 2024-12-05T12:13:56,912 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T12:13:56,914 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T12:13:56,914 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400836914"}]},"ts":"1733400836914"} 2024-12-05T12:13:56,916 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T12:13:56,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, ASSIGN}] 2024-12-05T12:13:56,918 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, ASSIGN 2024-12-05T12:13:56,919 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, ASSIGN; state=OFFLINE, location=9ea9abad0e9b,44759,1733400796278; forceNewPlan=false, retain=false 2024-12-05T12:13:57,069 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=dde686338ee027649f1546b042d1ca1b, regionState=OPENING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:57,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, ASSIGN because future has completed 2024-12-05T12:13:57,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:13:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T12:13:57,231 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:57,231 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:13:57,232 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,232 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:13:57,232 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,232 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,234 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,235 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:13:57,236 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde686338ee027649f1546b042d1ca1b columnFamilyName A 2024-12-05T12:13:57,237 DEBUG [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:57,237 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(327): Store=dde686338ee027649f1546b042d1ca1b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:57,237 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,239 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:13:57,239 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde686338ee027649f1546b042d1ca1b columnFamilyName B 2024-12-05T12:13:57,239 DEBUG [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:57,240 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(327): Store=dde686338ee027649f1546b042d1ca1b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:57,240 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,241 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:13:57,241 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde686338ee027649f1546b042d1ca1b columnFamilyName C 2024-12-05T12:13:57,241 DEBUG [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:57,242 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(327): Store=dde686338ee027649f1546b042d1ca1b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:57,242 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:57,242 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,243 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,243 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,244 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,244 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,245 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:13:57,246 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,249 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:13:57,249 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened dde686338ee027649f1546b042d1ca1b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64723230, jitterRate=-0.03554871678352356}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:13:57,249 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:57,250 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for dde686338ee027649f1546b042d1ca1b: Running coprocessor pre-open hook at 1733400837232Writing region info on filesystem at 1733400837232Initializing all the Stores at 1733400837233 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400837234 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400837234Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400837234Cleaning up temporary data from old regions at 1733400837244 (+10 ms)Running coprocessor post-open hooks at 1733400837249 (+5 ms)Region opened successfully at 1733400837250 (+1 ms) 2024-12-05T12:13:57,251 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., pid=36, masterSystemTime=1733400837225 2024-12-05T12:13:57,253 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:57,253 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:57,254 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=dde686338ee027649f1546b042d1ca1b, regionState=OPEN, openSeqNum=2, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:57,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:13:57,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-05T12:13:57,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 in 188 msec 2024-12-05T12:13:57,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-05T12:13:57,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, ASSIGN in 347 msec 2024-12-05T12:13:57,266 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T12:13:57,266 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400837266"}]},"ts":"1733400837266"} 2024-12-05T12:13:57,268 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T12:13:57,269 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T12:13:57,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-12-05T12:13:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T12:13:58,234 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:58,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e30321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:58,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:58,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:58,236 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:58,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:58,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5806ebd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:58,238 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d1de7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:58,239 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:58,241 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:58,243 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T12:13:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ea9abad0e9b,34103,1733400795471 2024-12-05T12:13:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f9f9fc2 2024-12-05T12:13:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T12:13:58,244 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T12:13:58,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T12:13:58,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:13:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T12:13:58,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741953_1129 (size=1097) 2024-12-05T12:13:58,675 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T12:13:58,675 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T12:13:58,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T12:13:58,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, REOPEN/MOVE}] 2024-12-05T12:13:58,686 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, REOPEN/MOVE 2024-12-05T12:13:58,687 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=dde686338ee027649f1546b042d1ca1b, regionState=CLOSING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:58,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, REOPEN/MOVE because future has completed 2024-12-05T12:13:58,689 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T12:13:58,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:13:58,841 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:58,841 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T12:13:58,841 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing dde686338ee027649f1546b042d1ca1b, disabling compactions & flushes 2024-12-05T12:13:58,841 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:58,841 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:58,841 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. after waiting 0 ms 2024-12-05T12:13:58,842 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:58,846 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T12:13:58,847 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:58,847 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for dde686338ee027649f1546b042d1ca1b: Waiting for close lock at 1733400838841Running coprocessor pre-close hooks at 1733400838841Disabling compacts and flushes for region at 1733400838841Disabling writes for close at 1733400838842 (+1 ms)Writing region close event to WAL at 1733400838842Running coprocessor post-close hooks at 1733400838846 (+4 ms)Closed at 1733400838846 2024-12-05T12:13:58,847 WARN [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: dde686338ee027649f1546b042d1ca1b to self. 2024-12-05T12:13:58,849 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:58,850 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=dde686338ee027649f1546b042d1ca1b, regionState=CLOSED 2024-12-05T12:13:58,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:13:58,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-05T12:13:58,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 in 165 msec 2024-12-05T12:13:58,858 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, REOPEN/MOVE; state=CLOSED, location=9ea9abad0e9b,44759,1733400796278; forceNewPlan=false, retain=true 2024-12-05T12:13:59,009 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=dde686338ee027649f1546b042d1ca1b, regionState=OPENING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, REOPEN/MOVE because future has completed 2024-12-05T12:13:59,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:13:59,168 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,168 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:13:59,169 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,169 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:13:59,169 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,169 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,174 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,175 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:13:59,180 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde686338ee027649f1546b042d1ca1b columnFamilyName A 2024-12-05T12:13:59,182 DEBUG [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:59,183 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(327): Store=dde686338ee027649f1546b042d1ca1b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:59,183 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,184 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:13:59,184 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde686338ee027649f1546b042d1ca1b columnFamilyName B 2024-12-05T12:13:59,185 DEBUG [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:59,185 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(327): Store=dde686338ee027649f1546b042d1ca1b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:59,185 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,186 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:13:59,186 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde686338ee027649f1546b042d1ca1b columnFamilyName C 2024-12-05T12:13:59,186 DEBUG [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:59,187 INFO [StoreOpener-dde686338ee027649f1546b042d1ca1b-1 {}] regionserver.HStore(327): Store=dde686338ee027649f1546b042d1ca1b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:13:59,187 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,187 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,188 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,190 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,191 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,191 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,191 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:13:59,193 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,194 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened dde686338ee027649f1546b042d1ca1b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63064531, jitterRate=-0.06026525795459747}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:13:59,194 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,196 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for dde686338ee027649f1546b042d1ca1b: Running coprocessor pre-open hook at 1733400839169Writing region info on filesystem at 1733400839169Initializing all the Stores at 1733400839170 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400839170Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400839170Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400839170Cleaning up temporary data from old regions at 1733400839191 (+21 ms)Running coprocessor post-open hooks at 1733400839194 (+3 ms)Region opened successfully at 1733400839195 (+1 ms) 2024-12-05T12:13:59,197 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., pid=41, masterSystemTime=1733400839164 2024-12-05T12:13:59,199 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,200 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,200 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=dde686338ee027649f1546b042d1ca1b, regionState=OPEN, openSeqNum=5, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:13:59,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-12-05T12:13:59,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 in 194 msec 2024-12-05T12:13:59,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-05T12:13:59,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, REOPEN/MOVE in 523 msec 2024-12-05T12:13:59,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-05T12:13:59,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 535 msec 2024-12-05T12:13:59,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 965 msec 2024-12-05T12:13:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-05T12:13:59,227 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:13:59,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9c3dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:59,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:59,230 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d13c98e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:59,232 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259191ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dde8260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:59,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:59,243 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:59,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:59,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:59,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a2ccb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:59,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:59,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:59,247 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e7ab686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab5c81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:59,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:59,251 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:59,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:59,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:59,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf99e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:59,253 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5ab380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1a430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:59,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:59,257 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@757255aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:59,261 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6971ff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbff363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:59,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:59,266 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@628882f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:59,269 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b08f6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292bddff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:59,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:59,274 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fbcd529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:59,283 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:59,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38267996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a14e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:59,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:59,286 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd8da48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:59,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8f24a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f17c234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:59,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:59,290 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3335bcf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:59,292 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f74fba5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197f7df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:13:59,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:13:59,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:13:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:13:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:13:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1111c992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:13:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:13:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:13:59,297 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:13:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c97c563, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:13:59,299 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:59,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:13:59,299 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:59,299 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:59,299 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:59,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:59,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:59,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:59,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:59,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:59,307 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:59,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-05T12:13:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:13:59,309 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:13:59,310 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:13:59,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:13:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:13:59,311 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:13:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T12:13:59,313 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:59,313 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:59,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:59,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:13:59,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:13:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:13:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:13:59,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:59,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:13:59,343 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:13:59,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:13:59,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:13:59,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:13:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:13:59,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:13:59,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:13:59,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:59,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:13:59,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:59,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:13:59,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:13:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T12:13:59,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400839370, max timestamp=1733400839422Num uniques 859; 2024-12-05T12:13:59,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:13:59,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120547a3d211402e4c3b91dc51cbe5df0fec_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400839351/Put/seqid=0 2024-12-05T12:13:59,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400839370, max timestamp=1733400839422Num uniques 859; 2024-12-05T12:13:59,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:13:59,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400839370, max timestamp=1733400839422Num uniques 859; 2024-12-05T12:13:59,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:13:59,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T12:13:59,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:13:59,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400899476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741954_1130 (size=12154) 2024-12-05T12:13:59,491 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:13:59,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400899477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400899478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400899480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120547a3d211402e4c3b91dc51cbe5df0fec_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120547a3d211402e4c3b91dc51cbe5df0fec_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:13:59,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400899487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/5e3bfdf189494a148e4028aeacfaef5a, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:13:59,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/5e3bfdf189494a148e4028aeacfaef5a is 175, key is test_row_0/A:col10/1733400839351/Put/seqid=0 2024-12-05T12:13:59,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741955_1131 (size=30955) 2024-12-05T12:13:59,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/5e3bfdf189494a148e4028aeacfaef5a 2024-12-05T12:13:59,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400899593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400899603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T12:13:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:13:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400899616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400899616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400899619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/935ba20078b148abafcb6c58962bca2d is 50, key is test_row_0/B:col10/1733400839351/Put/seqid=0 2024-12-05T12:13:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T12:13:59,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741956_1132 (size=12001) 2024-12-05T12:13:59,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/935ba20078b148abafcb6c58962bca2d 2024-12-05T12:13:59,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/3aa587ee12d94dfb93118a95e7c8521a is 50, key is test_row_0/C:col10/1733400839351/Put/seqid=0 2024-12-05T12:13:59,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741957_1133 (size=12001) 2024-12-05T12:13:59,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T12:13:59,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:13:59,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400899817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400899819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400899833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400899835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:13:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400899836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:13:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:13:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:13:59,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T12:13:59,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:13:59,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:13:59,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:13:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T12:14:00,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T12:14:00,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:00,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:00,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:00,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:00,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:00,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400900139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400900139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400900144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/3aa587ee12d94dfb93118a95e7c8521a 2024-12-05T12:14:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400900144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400900163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/5e3bfdf189494a148e4028aeacfaef5a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5e3bfdf189494a148e4028aeacfaef5a 2024-12-05T12:14:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5e3bfdf189494a148e4028aeacfaef5a, entries=150, sequenceid=14, filesize=30.2 K 2024-12-05T12:14:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/935ba20078b148abafcb6c58962bca2d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/935ba20078b148abafcb6c58962bca2d 2024-12-05T12:14:00,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/935ba20078b148abafcb6c58962bca2d, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T12:14:00,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/3aa587ee12d94dfb93118a95e7c8521a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3aa587ee12d94dfb93118a95e7c8521a 2024-12-05T12:14:00,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3aa587ee12d94dfb93118a95e7c8521a, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T12:14:00,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for dde686338ee027649f1546b042d1ca1b in 870ms, sequenceid=14, compaction requested=false 2024-12-05T12:14:00,226 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T12:14:00,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:00,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T12:14:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:00,256 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-05T12:14:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:00,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205dbe40f9f3c204ed1825f14854b1280d0_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400839471/Put/seqid=0 2024-12-05T12:14:00,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741958_1134 (size=12154) 2024-12-05T12:14:00,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:00,355 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205dbe40f9f3c204ed1825f14854b1280d0_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205dbe40f9f3c204ed1825f14854b1280d0_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/3c815f3a2db342e5aa97ec54cfe3db92, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/3c815f3a2db342e5aa97ec54cfe3db92 is 175, key is test_row_0/A:col10/1733400839471/Put/seqid=0 2024-12-05T12:14:00,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741959_1135 (size=30955) 2024-12-05T12:14:00,412 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/3c815f3a2db342e5aa97ec54cfe3db92 2024-12-05T12:14:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T12:14:00,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/78965755cf0b4ab4a6f8ad352c91bb3c is 50, key is test_row_0/B:col10/1733400839471/Put/seqid=0 2024-12-05T12:14:00,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741960_1136 (size=12001) 2024-12-05T12:14:00,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:00,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400900684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400900689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400900692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400900698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400900699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,784 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T12:14:00,810 INFO [master/9ea9abad0e9b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T12:14:00,811 INFO [master/9ea9abad0e9b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T12:14:00,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400900803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400900817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400900818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400900818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:00,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:00,935 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/78965755cf0b4ab4a6f8ad352c91bb3c 2024-12-05T12:14:00,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/a2833617a77344c3ad545497c6fb016c is 50, key is test_row_0/C:col10/1733400839471/Put/seqid=0 2024-12-05T12:14:00,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741961_1137 (size=12001) 2024-12-05T12:14:00,998 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/a2833617a77344c3ad545497c6fb016c 2024-12-05T12:14:01,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/3c815f3a2db342e5aa97ec54cfe3db92 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3c815f3a2db342e5aa97ec54cfe3db92 2024-12-05T12:14:01,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3c815f3a2db342e5aa97ec54cfe3db92, entries=150, sequenceid=50, filesize=30.2 K 2024-12-05T12:14:01,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400901037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/78965755cf0b4ab4a6f8ad352c91bb3c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/78965755cf0b4ab4a6f8ad352c91bb3c 2024-12-05T12:14:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400901038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400901038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400901043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,071 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/78965755cf0b4ab4a6f8ad352c91bb3c, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T12:14:01,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/a2833617a77344c3ad545497c6fb016c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a2833617a77344c3ad545497c6fb016c 2024-12-05T12:14:01,079 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a2833617a77344c3ad545497c6fb016c, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T12:14:01,086 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for dde686338ee027649f1546b042d1ca1b in 830ms, sequenceid=50, compaction requested=false 2024-12-05T12:14:01,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:01,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-05T12:14:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-05T12:14:01,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-05T12:14:01,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7810 sec 2024-12-05T12:14:01,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7970 sec 2024-12-05T12:14:01,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:14:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:01,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:01,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:01,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:01,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:01,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:01,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:01,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057d60d0a95fe644bbb3a5d808f6225a98_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400840691/Put/seqid=0 2024-12-05T12:14:01,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741962_1138 (size=14594) 2024-12-05T12:14:01,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400841403, max timestamp=1733400841444Num uniques 859; 2024-12-05T12:14:01,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:14:01,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400841403, max timestamp=1733400841444Num uniques 859; 2024-12-05T12:14:01,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:01,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400841403, max timestamp=1733400841448Num uniques 859; 2024-12-05T12:14:01,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T12:14:01,464 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:01,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-05T12:14:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T12:14:01,468 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:01,469 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:01,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:01,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400901548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400901541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400901565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400901570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T12:14:01,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T12:14:01,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:01,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:01,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:01,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:01,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:01,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400901673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400901683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400901683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400901683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400901714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,720 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T12:14:01,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T12:14:01,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:01,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:01,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:01,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:01,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:01,828 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:01,833 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057d60d0a95fe644bbb3a5d808f6225a98_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057d60d0a95fe644bbb3a5d808f6225a98_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:01,834 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/acd48bf99ecc4f40bf8c36fed29a2ea3, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:01,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/acd48bf99ecc4f40bf8c36fed29a2ea3 is 175, key is test_row_0/A:col10/1733400840691/Put/seqid=0 2024-12-05T12:14:01,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741963_1139 (size=39549) 2024-12-05T12:14:01,862 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/acd48bf99ecc4f40bf8c36fed29a2ea3 2024-12-05T12:14:01,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400901883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b7a451868de94989be9aba0c8d9bb974 is 50, key is test_row_0/B:col10/1733400840691/Put/seqid=0 2024-12-05T12:14:01,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400901903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400901903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400901903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:01,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741964_1140 (size=12001) 2024-12-05T12:14:01,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T12:14:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:01,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T12:14:02,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T12:14:02,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:02,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:02,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:02,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:02,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:02,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400902193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:02,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400902227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:02,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400902227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400902227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:02,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T12:14:02,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:02,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:02,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:02,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:02,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:02,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b7a451868de94989be9aba0c8d9bb974 2024-12-05T12:14:02,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/8d970355a78d40b2ba2f22fb166a16c6 is 50, key is test_row_0/C:col10/1733400840691/Put/seqid=0 2024-12-05T12:14:02,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741965_1141 (size=12001) 2024-12-05T12:14:02,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/8d970355a78d40b2ba2f22fb166a16c6 2024-12-05T12:14:02,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/acd48bf99ecc4f40bf8c36fed29a2ea3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/acd48bf99ecc4f40bf8c36fed29a2ea3 2024-12-05T12:14:02,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/acd48bf99ecc4f40bf8c36fed29a2ea3, entries=200, sequenceid=61, filesize=38.6 K 2024-12-05T12:14:02,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b7a451868de94989be9aba0c8d9bb974 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b7a451868de94989be9aba0c8d9bb974 2024-12-05T12:14:02,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b7a451868de94989be9aba0c8d9bb974, entries=150, sequenceid=61, filesize=11.7 K 2024-12-05T12:14:02,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/8d970355a78d40b2ba2f22fb166a16c6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d970355a78d40b2ba2f22fb166a16c6 2024-12-05T12:14:02,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d970355a78d40b2ba2f22fb166a16c6, entries=150, sequenceid=61, filesize=11.7 K 2024-12-05T12:14:02,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for dde686338ee027649f1546b042d1ca1b in 1061ms, sequenceid=61, compaction requested=true 2024-12-05T12:14:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:02,422 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:02,423 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:02,424 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:02,424 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:02,424 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/A is initiating minor compaction (all files) 2024-12-05T12:14:02,424 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/B is initiating minor compaction (all files) 2024-12-05T12:14:02,425 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/B in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:02,425 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/935ba20078b148abafcb6c58962bca2d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/78965755cf0b4ab4a6f8ad352c91bb3c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b7a451868de94989be9aba0c8d9bb974] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=35.2 K 2024-12-05T12:14:02,425 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/A in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:02,426 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5e3bfdf189494a148e4028aeacfaef5a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3c815f3a2db342e5aa97ec54cfe3db92, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/acd48bf99ecc4f40bf8c36fed29a2ea3] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=99.1 K 2024-12-05T12:14:02,426 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:02,426 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5e3bfdf189494a148e4028aeacfaef5a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3c815f3a2db342e5aa97ec54cfe3db92, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/acd48bf99ecc4f40bf8c36fed29a2ea3] 2024-12-05T12:14:02,426 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 935ba20078b148abafcb6c58962bca2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733400839329 2024-12-05T12:14:02,433 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e3bfdf189494a148e4028aeacfaef5a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733400839329 2024-12-05T12:14:02,434 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c815f3a2db342e5aa97ec54cfe3db92, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733400839471 2024-12-05T12:14:02,435 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting acd48bf99ecc4f40bf8c36fed29a2ea3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400840687 2024-12-05T12:14:02,435 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 78965755cf0b4ab4a6f8ad352c91bb3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733400839471 2024-12-05T12:14:02,436 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a451868de94989be9aba0c8d9bb974, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400840691 2024-12-05T12:14:02,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T12:14:02,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:02,450 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-12-05T12:14:02,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:02,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:02,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:02,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:02,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:02,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:02,463 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:02,472 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#B#compaction#124 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:02,473 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/eb442a5ebefe499bbd86a50bd3f9fd3a is 50, key is test_row_0/B:col10/1733400840691/Put/seqid=0 2024-12-05T12:14:02,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b79deaa4d68749f788e8ea745b96a3cc_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400841558/Put/seqid=0 2024-12-05T12:14:02,477 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412052f80c698b33c42b787c836bd394d0311_dde686338ee027649f1546b042d1ca1b store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:02,492 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412052f80c698b33c42b787c836bd394d0311_dde686338ee027649f1546b042d1ca1b, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:02,492 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052f80c698b33c42b787c836bd394d0311_dde686338ee027649f1546b042d1ca1b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:02,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741966_1142 (size=12104) 2024-12-05T12:14:02,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741967_1143 (size=12154) 2024-12-05T12:14:02,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:02,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741968_1144 (size=4469) 2024-12-05T12:14:02,534 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#A#compaction#123 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:02,535 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/154544d526a6427da6e0acd90f3a3076 is 175, key is test_row_0/A:col10/1733400840691/Put/seqid=0 2024-12-05T12:14:02,541 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b79deaa4d68749f788e8ea745b96a3cc_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b79deaa4d68749f788e8ea745b96a3cc_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:02,542 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/eb442a5ebefe499bbd86a50bd3f9fd3a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/eb442a5ebefe499bbd86a50bd3f9fd3a 2024-12-05T12:14:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/e69d1515358248d68efe2c723a38989c, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/e69d1515358248d68efe2c723a38989c is 175, key is test_row_0/A:col10/1733400841558/Put/seqid=0 2024-12-05T12:14:02,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741970_1146 (size=30955) 2024-12-05T12:14:02,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741969_1145 (size=31058) 2024-12-05T12:14:02,560 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/B of dde686338ee027649f1546b042d1ca1b into eb442a5ebefe499bbd86a50bd3f9fd3a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:02,560 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:02,561 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/B, priority=13, startTime=1733400842423; duration=0sec 2024-12-05T12:14:02,561 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:02,561 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:B 2024-12-05T12:14:02,561 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:02,562 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:02,563 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/C is initiating minor compaction (all files) 2024-12-05T12:14:02,563 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/C in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:02,563 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3aa587ee12d94dfb93118a95e7c8521a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a2833617a77344c3ad545497c6fb016c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d970355a78d40b2ba2f22fb166a16c6] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=35.2 K 2024-12-05T12:14:02,564 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aa587ee12d94dfb93118a95e7c8521a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733400839329 2024-12-05T12:14:02,565 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting a2833617a77344c3ad545497c6fb016c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733400839471 2024-12-05T12:14:02,566 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d970355a78d40b2ba2f22fb166a16c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400840691 2024-12-05T12:14:02,580 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:02,581 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/e7b8e6bc001f4a7aa2bc4d44e8a3b961 is 50, key is test_row_0/C:col10/1733400840691/Put/seqid=0 2024-12-05T12:14:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T12:14:02,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741971_1147 (size=12104) 2024-12-05T12:14:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:02,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:02,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400902743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:02,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400902743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:02,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400902744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:02,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:02,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400902747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:02,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:02,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400902867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400902867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:02,952 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/e69d1515358248d68efe2c723a38989c 2024-12-05T12:14:02,962 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/154544d526a6427da6e0acd90f3a3076 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/154544d526a6427da6e0acd90f3a3076 2024-12-05T12:14:02,975 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/A of dde686338ee027649f1546b042d1ca1b into 154544d526a6427da6e0acd90f3a3076(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:02,975 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:02,975 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/A, priority=13, startTime=1733400842422; duration=0sec 2024-12-05T12:14:02,976 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:02,976 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:A 2024-12-05T12:14:02,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/41f5d64064164d53b22ad8a4697e8a03 is 50, key is test_row_0/B:col10/1733400841558/Put/seqid=0 2024-12-05T12:14:03,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741972_1148 (size=12001) 2024-12-05T12:14:03,038 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/41f5d64064164d53b22ad8a4697e8a03 2024-12-05T12:14:03,046 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/e7b8e6bc001f4a7aa2bc4d44e8a3b961 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/e7b8e6bc001f4a7aa2bc4d44e8a3b961 2024-12-05T12:14:03,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/54165a72b9a748b9bba8454be91af37b is 50, key is test_row_0/C:col10/1733400841558/Put/seqid=0 2024-12-05T12:14:03,058 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/C of dde686338ee027649f1546b042d1ca1b into e7b8e6bc001f4a7aa2bc4d44e8a3b961(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:03,058 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:03,059 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/C, priority=13, startTime=1733400842423; duration=0sec 2024-12-05T12:14:03,059 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:03,059 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:C 2024-12-05T12:14:03,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400903083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:03,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400903093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:03,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741973_1149 (size=12001) 2024-12-05T12:14:03,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400903403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:03,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400903412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:03,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:03,525 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/54165a72b9a748b9bba8454be91af37b 2024-12-05T12:14:03,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/e69d1515358248d68efe2c723a38989c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e69d1515358248d68efe2c723a38989c 2024-12-05T12:14:03,538 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e69d1515358248d68efe2c723a38989c, entries=150, sequenceid=98, filesize=30.2 K 2024-12-05T12:14:03,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/41f5d64064164d53b22ad8a4697e8a03 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/41f5d64064164d53b22ad8a4697e8a03 2024-12-05T12:14:03,548 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/41f5d64064164d53b22ad8a4697e8a03, entries=150, sequenceid=98, filesize=11.7 K 2024-12-05T12:14:03,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/54165a72b9a748b9bba8454be91af37b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/54165a72b9a748b9bba8454be91af37b 2024-12-05T12:14:03,560 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/54165a72b9a748b9bba8454be91af37b, entries=150, sequenceid=98, filesize=11.7 K 2024-12-05T12:14:03,561 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for dde686338ee027649f1546b042d1ca1b in 1111ms, sequenceid=98, compaction requested=false 2024-12-05T12:14:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-05T12:14:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-05T12:14:03,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-05T12:14:03,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0940 sec 2024-12-05T12:14:03,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.1020 sec 2024-12-05T12:14:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T12:14:03,613 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:03,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-05T12:14:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T12:14:03,617 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:03,618 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:03,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T12:14:03,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T12:14:03,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:03,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:03,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:03,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:03,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:03,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:03,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120568072543768e4321abcf6a12cdf218ba_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400842745/Put/seqid=0 2024-12-05T12:14:03,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741974_1150 (size=14594) 2024-12-05T12:14:03,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T12:14:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:03,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:03,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:03,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:03,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:03,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400843785, max timestamp=1733400843796Num uniques 859; 2024-12-05T12:14:03,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400843785, max timestamp=1733400843796Num uniques 859; 2024-12-05T12:14:03,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:14:03,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:14:03,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400843785, max timestamp=1733400843796Num uniques 859; 2024-12-05T12:14:03,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:14:03,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400903876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:03,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400903879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:03,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400903885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:03,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400903913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:03,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T12:14:03,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:03,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:03,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:03,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:03,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T12:14:03,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400903923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:03,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400903993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:04,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400904002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:04,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:04,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400904013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:04,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T12:14:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,164 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:04,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120568072543768e4321abcf6a12cdf218ba_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120568072543768e4321abcf6a12cdf218ba_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:04,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/0e795c8d66a24b65b24e313220274d60, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:04,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/0e795c8d66a24b65b24e313220274d60 is 175, key is test_row_0/A:col10/1733400842745/Put/seqid=0 2024-12-05T12:14:04,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741975_1151 (size=39549) 2024-12-05T12:14:04,205 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/0e795c8d66a24b65b24e313220274d60 2024-12-05T12:14:04,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400904202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:04,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:04,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400904213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:04,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/f653b7fb38b648929402f783c6e78ff8 is 50, key is test_row_0/B:col10/1733400842745/Put/seqid=0 2024-12-05T12:14:04,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400904223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:04,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T12:14:04,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:04,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T12:14:04,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741976_1152 (size=12001) 2024-12-05T12:14:04,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/f653b7fb38b648929402f783c6e78ff8 2024-12-05T12:14:04,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/ea23cf21764c426aa8409da030e1fbb7 is 50, key is test_row_0/C:col10/1733400842745/Put/seqid=0 2024-12-05T12:14:04,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741977_1153 (size=12001) 2024-12-05T12:14:04,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T12:14:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400904513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:04,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400904523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:04,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400904542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:04,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:04,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T12:14:04,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:04,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T12:14:04,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:04,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:04,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/ea23cf21764c426aa8409da030e1fbb7 2024-12-05T12:14:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/0e795c8d66a24b65b24e313220274d60 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/0e795c8d66a24b65b24e313220274d60 2024-12-05T12:14:04,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/0e795c8d66a24b65b24e313220274d60, entries=200, sequenceid=112, filesize=38.6 K 2024-12-05T12:14:04,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/f653b7fb38b648929402f783c6e78ff8 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/f653b7fb38b648929402f783c6e78ff8 2024-12-05T12:14:04,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/f653b7fb38b648929402f783c6e78ff8, entries=150, sequenceid=112, filesize=11.7 K 2024-12-05T12:14:04,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/ea23cf21764c426aa8409da030e1fbb7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ea23cf21764c426aa8409da030e1fbb7 2024-12-05T12:14:04,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ea23cf21764c426aa8409da030e1fbb7, entries=150, sequenceid=112, filesize=11.7 K 2024-12-05T12:14:04,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dde686338ee027649f1546b042d1ca1b in 1020ms, sequenceid=112, compaction requested=true 2024-12-05T12:14:04,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:04,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:04,746 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:04,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:04,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:04,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:04,746 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:04,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:04,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:04,748 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:04,748 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/B is initiating minor compaction (all files) 2024-12-05T12:14:04,748 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/B in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,748 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/eb442a5ebefe499bbd86a50bd3f9fd3a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/41f5d64064164d53b22ad8a4697e8a03, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/f653b7fb38b648929402f783c6e78ff8] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=35.3 K 2024-12-05T12:14:04,748 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:04,748 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/A is initiating minor compaction (all files) 2024-12-05T12:14:04,748 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/A in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,748 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/154544d526a6427da6e0acd90f3a3076, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e69d1515358248d68efe2c723a38989c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/0e795c8d66a24b65b24e313220274d60] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=99.2 K 2024-12-05T12:14:04,748 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,749 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/154544d526a6427da6e0acd90f3a3076, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e69d1515358248d68efe2c723a38989c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/0e795c8d66a24b65b24e313220274d60] 2024-12-05T12:14:04,749 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting eb442a5ebefe499bbd86a50bd3f9fd3a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400840691 2024-12-05T12:14:04,749 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 154544d526a6427da6e0acd90f3a3076, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400840691 2024-12-05T12:14:04,750 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 41f5d64064164d53b22ad8a4697e8a03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733400841537 2024-12-05T12:14:04,750 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting e69d1515358248d68efe2c723a38989c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733400841537 2024-12-05T12:14:04,750 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e795c8d66a24b65b24e313220274d60, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400842727 2024-12-05T12:14:04,751 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting f653b7fb38b648929402f783c6e78ff8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400842727 2024-12-05T12:14:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T12:14:04,763 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:04,771 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#B#compaction#132 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:04,772 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b116511ae15c4c319fa3c68a74555dce is 50, key is test_row_0/B:col10/1733400842745/Put/seqid=0 2024-12-05T12:14:04,772 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412053e6eee689cfd4a608064148e3bc22b2c_dde686338ee027649f1546b042d1ca1b store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:04,775 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412053e6eee689cfd4a608064148e3bc22b2c_dde686338ee027649f1546b042d1ca1b, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:04,775 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053e6eee689cfd4a608064148e3bc22b2c_dde686338ee027649f1546b042d1ca1b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:04,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741978_1154 (size=12207) 2024-12-05T12:14:04,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741979_1155 (size=4469) 2024-12-05T12:14:04,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T12:14:04,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:04,852 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-05T12:14:04,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:04,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:04,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:04,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:04,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:04,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:04,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205973517785e394cfaad005a2c6b85cdc8_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400843871/Put/seqid=0 2024-12-05T12:14:04,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741980_1156 (size=12304) 2024-12-05T12:14:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:04,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:04,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400904947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:04,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:04,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400904953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:04,956 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:05,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400905022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:05,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400905033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:05,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:05,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400905053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400905062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:05,190 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b116511ae15c4c319fa3c68a74555dce as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b116511ae15c4c319fa3c68a74555dce 2024-12-05T12:14:05,193 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#A#compaction#133 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:05,194 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/000c9526dbfa477283788dfd1f7a85f2 is 175, key is test_row_0/A:col10/1733400842745/Put/seqid=0 2024-12-05T12:14:05,198 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/B of dde686338ee027649f1546b042d1ca1b into b116511ae15c4c319fa3c68a74555dce(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:05,198 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:05,198 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/B, priority=13, startTime=1733400844746; duration=0sec 2024-12-05T12:14:05,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741981_1157 (size=31161) 2024-12-05T12:14:05,198 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:05,198 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:B 2024-12-05T12:14:05,198 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:05,200 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:05,200 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/C is initiating minor compaction (all files) 2024-12-05T12:14:05,200 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/C in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:05,200 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/e7b8e6bc001f4a7aa2bc4d44e8a3b961, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/54165a72b9a748b9bba8454be91af37b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ea23cf21764c426aa8409da030e1fbb7] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=35.3 K 2024-12-05T12:14:05,200 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting e7b8e6bc001f4a7aa2bc4d44e8a3b961, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400840691 2024-12-05T12:14:05,201 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 54165a72b9a748b9bba8454be91af37b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733400841537 2024-12-05T12:14:05,201 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting ea23cf21764c426aa8409da030e1fbb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400842727 2024-12-05T12:14:05,204 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/000c9526dbfa477283788dfd1f7a85f2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/000c9526dbfa477283788dfd1f7a85f2 2024-12-05T12:14:05,210 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/A of dde686338ee027649f1546b042d1ca1b into 000c9526dbfa477283788dfd1f7a85f2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:05,210 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:05,210 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/A, priority=13, startTime=1733400844746; duration=0sec 2024-12-05T12:14:05,210 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:05,210 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:A 2024-12-05T12:14:05,214 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#C#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:05,214 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/8d34f34bc07e4b26b32e38ed9994bbab is 50, key is test_row_0/C:col10/1733400842745/Put/seqid=0 2024-12-05T12:14:05,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741982_1158 (size=12207) 2024-12-05T12:14:05,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:05,271 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205973517785e394cfaad005a2c6b85cdc8_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205973517785e394cfaad005a2c6b85cdc8_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:05,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/c2e3dd708f8149c9a845535bc448071d, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:05,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/c2e3dd708f8149c9a845535bc448071d is 175, key is test_row_0/A:col10/1733400843871/Put/seqid=0 2024-12-05T12:14:05,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400905273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:05,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741983_1159 (size=31105) 2024-12-05T12:14:05,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/c2e3dd708f8149c9a845535bc448071d 2024-12-05T12:14:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/84b7a8b0d0304e19a1ceb12af065820d is 50, key is test_row_0/B:col10/1733400843871/Put/seqid=0 2024-12-05T12:14:05,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741984_1160 (size=12151) 2024-12-05T12:14:05,325 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/84b7a8b0d0304e19a1ceb12af065820d 2024-12-05T12:14:05,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/a17f06ad94494e869e70b5f2571ee6a1 is 50, key is test_row_0/C:col10/1733400843871/Put/seqid=0 2024-12-05T12:14:05,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741985_1161 (size=12151) 2024-12-05T12:14:05,340 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/a17f06ad94494e869e70b5f2571ee6a1 2024-12-05T12:14:05,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/c2e3dd708f8149c9a845535bc448071d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c2e3dd708f8149c9a845535bc448071d 2024-12-05T12:14:05,362 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c2e3dd708f8149c9a845535bc448071d, entries=150, sequenceid=146, filesize=30.4 K 2024-12-05T12:14:05,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/84b7a8b0d0304e19a1ceb12af065820d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/84b7a8b0d0304e19a1ceb12af065820d 2024-12-05T12:14:05,369 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/84b7a8b0d0304e19a1ceb12af065820d, entries=150, sequenceid=146, filesize=11.9 K 2024-12-05T12:14:05,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/a17f06ad94494e869e70b5f2571ee6a1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a17f06ad94494e869e70b5f2571ee6a1 2024-12-05T12:14:05,376 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a17f06ad94494e869e70b5f2571ee6a1, entries=150, sequenceid=146, filesize=11.9 K 2024-12-05T12:14:05,378 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for dde686338ee027649f1546b042d1ca1b in 526ms, sequenceid=146, compaction requested=false 2024-12-05T12:14:05,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:05,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:05,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-05T12:14:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-05T12:14:05,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-05T12:14:05,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7620 sec 2024-12-05T12:14:05,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.7690 sec 2024-12-05T12:14:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:05,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T12:14:05,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:05,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:05,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:05,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:05,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:05,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:05,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d5456588ddb141ad9cce64c99e3e161f_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400844946/Put/seqid=0 2024-12-05T12:14:05,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741986_1162 (size=14794) 2024-12-05T12:14:05,626 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:05,636 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d5456588ddb141ad9cce64c99e3e161f_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d5456588ddb141ad9cce64c99e3e161f_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:05,637 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/8d34f34bc07e4b26b32e38ed9994bbab as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d34f34bc07e4b26b32e38ed9994bbab 2024-12-05T12:14:05,638 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/39e6eaf74d114b91a64f858af878422e, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:05,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/39e6eaf74d114b91a64f858af878422e is 175, key is test_row_0/A:col10/1733400844946/Put/seqid=0 2024-12-05T12:14:05,647 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/C of dde686338ee027649f1546b042d1ca1b into 8d34f34bc07e4b26b32e38ed9994bbab(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:05,647 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:05,647 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/C, priority=13, startTime=1733400844746; duration=0sec 2024-12-05T12:14:05,647 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:05,647 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:C 2024-12-05T12:14:05,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741987_1163 (size=39749) 2024-12-05T12:14:05,653 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/39e6eaf74d114b91a64f858af878422e 2024-12-05T12:14:05,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b09171de9ddc4efb8200debf0cb973d0 is 50, key is test_row_0/B:col10/1733400844946/Put/seqid=0 2024-12-05T12:14:05,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400905668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:05,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741988_1164 (size=12151) 2024-12-05T12:14:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T12:14:05,763 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:05,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-05T12:14:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T12:14:05,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:05,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:05,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:05,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400905783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T12:14:05,895 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T12:14:05,895 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T12:14:05,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T12:14:05,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:05,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:05,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:05,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:05,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:05,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400905993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:06,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400906043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:06,044 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:06,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400906052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:06,054 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:06,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T12:14:06,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:06,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:06,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:06,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400906072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:06,074 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:06,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b09171de9ddc4efb8200debf0cb973d0 2024-12-05T12:14:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T12:14:06,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/19564e41352e4307bb568bf327388554 is 50, key is test_row_0/C:col10/1733400844946/Put/seqid=0 2024-12-05T12:14:06,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741989_1165 (size=12151) 2024-12-05T12:14:06,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T12:14:06,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:06,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:06,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400906302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:06,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T12:14:06,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:06,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:06,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T12:14:06,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/19564e41352e4307bb568bf327388554 2024-12-05T12:14:06,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/39e6eaf74d114b91a64f858af878422e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/39e6eaf74d114b91a64f858af878422e 2024-12-05T12:14:06,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/39e6eaf74d114b91a64f858af878422e, entries=200, sequenceid=160, filesize=38.8 K 2024-12-05T12:14:06,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b09171de9ddc4efb8200debf0cb973d0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b09171de9ddc4efb8200debf0cb973d0 2024-12-05T12:14:06,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b09171de9ddc4efb8200debf0cb973d0, entries=150, sequenceid=160, filesize=11.9 K 2024-12-05T12:14:06,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/19564e41352e4307bb568bf327388554 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/19564e41352e4307bb568bf327388554 2024-12-05T12:14:06,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/19564e41352e4307bb568bf327388554, entries=150, sequenceid=160, filesize=11.9 K 2024-12-05T12:14:06,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for dde686338ee027649f1546b042d1ca1b in 923ms, sequenceid=160, compaction requested=true 2024-12-05T12:14:06,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:06,510 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:06,510 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:06,511 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:06,511 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:06,511 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/A is initiating minor compaction (all files) 2024-12-05T12:14:06,511 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/B is initiating minor compaction (all files) 2024-12-05T12:14:06,511 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/A in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,511 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/B in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,511 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/000c9526dbfa477283788dfd1f7a85f2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c2e3dd708f8149c9a845535bc448071d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/39e6eaf74d114b91a64f858af878422e] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=99.6 K 2024-12-05T12:14:06,511 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b116511ae15c4c319fa3c68a74555dce, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/84b7a8b0d0304e19a1ceb12af065820d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b09171de9ddc4efb8200debf0cb973d0] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=35.7 K 2024-12-05T12:14:06,511 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,511 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/000c9526dbfa477283788dfd1f7a85f2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c2e3dd708f8149c9a845535bc448071d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/39e6eaf74d114b91a64f858af878422e] 2024-12-05T12:14:06,511 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b116511ae15c4c319fa3c68a74555dce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400842727 2024-12-05T12:14:06,511 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 000c9526dbfa477283788dfd1f7a85f2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400842727 2024-12-05T12:14:06,512 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84b7a8b0d0304e19a1ceb12af065820d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733400843866 2024-12-05T12:14:06,512 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting c2e3dd708f8149c9a845535bc448071d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733400843866 2024-12-05T12:14:06,512 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b09171de9ddc4efb8200debf0cb973d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733400844944 2024-12-05T12:14:06,512 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 39e6eaf74d114b91a64f858af878422e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733400844941 2024-12-05T12:14:06,518 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:06,519 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/864b3679c3524b8caa0924854bea028a is 50, key is test_row_0/B:col10/1733400844946/Put/seqid=0 2024-12-05T12:14:06,520 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:06,523 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120520d3bc510856481aa1d77827153e863f_dde686338ee027649f1546b042d1ca1b store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:06,525 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120520d3bc510856481aa1d77827153e863f_dde686338ee027649f1546b042d1ca1b, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:06,526 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120520d3bc510856481aa1d77827153e863f_dde686338ee027649f1546b042d1ca1b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:06,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741990_1166 (size=12459) 2024-12-05T12:14:06,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T12:14:06,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,531 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T12:14:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:06,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741991_1167 (size=4469) 2024-12-05T12:14:06,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205641dc62b1b8b4fa9b15819bfffc3cf39_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400845657/Put/seqid=0 2024-12-05T12:14:06,541 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#A#compaction#142 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:06,542 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/ecac83d0bba44e0c876b9486265f5998 is 175, key is test_row_0/A:col10/1733400844946/Put/seqid=0 2024-12-05T12:14:06,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741992_1168 (size=12304) 2024-12-05T12:14:06,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741993_1169 (size=31413) 2024-12-05T12:14:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:06,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:06,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400906842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T12:14:06,936 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/864b3679c3524b8caa0924854bea028a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/864b3679c3524b8caa0924854bea028a 2024-12-05T12:14:06,941 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/B of dde686338ee027649f1546b042d1ca1b into 864b3679c3524b8caa0924854bea028a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:06,941 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:06,941 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/B, priority=13, startTime=1733400846510; duration=0sec 2024-12-05T12:14:06,941 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:06,941 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:B 2024-12-05T12:14:06,941 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:06,943 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:06,943 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/C is initiating minor compaction (all files) 2024-12-05T12:14:06,943 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/C in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:06,943 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d34f34bc07e4b26b32e38ed9994bbab, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a17f06ad94494e869e70b5f2571ee6a1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/19564e41352e4307bb568bf327388554] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=35.7 K 2024-12-05T12:14:06,945 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d34f34bc07e4b26b32e38ed9994bbab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400842727 2024-12-05T12:14:06,946 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting a17f06ad94494e869e70b5f2571ee6a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733400843866 2024-12-05T12:14:06,946 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19564e41352e4307bb568bf327388554, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733400844944 2024-12-05T12:14:06,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:06,953 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:06,954 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/3880494075004290aef17a068857dc1f is 50, key is test_row_0/C:col10/1733400844946/Put/seqid=0 2024-12-05T12:14:06,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400906952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:06,957 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205641dc62b1b8b4fa9b15819bfffc3cf39_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205641dc62b1b8b4fa9b15819bfffc3cf39_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:06,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/70bef445702745979a2954bea8277de3, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:06,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/70bef445702745979a2954bea8277de3 is 175, key is test_row_0/A:col10/1733400845657/Put/seqid=0 2024-12-05T12:14:06,962 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/ecac83d0bba44e0c876b9486265f5998 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/ecac83d0bba44e0c876b9486265f5998 2024-12-05T12:14:06,969 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/A of dde686338ee027649f1546b042d1ca1b into ecac83d0bba44e0c876b9486265f5998(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:06,969 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:06,969 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/A, priority=13, startTime=1733400846510; duration=0sec 2024-12-05T12:14:06,970 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:06,970 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:A 2024-12-05T12:14:06,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400906973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:06,975 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:06,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741995_1171 (size=31105) 2024-12-05T12:14:06,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741994_1170 (size=12459) 2024-12-05T12:14:07,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400907163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:07,383 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/70bef445702745979a2954bea8277de3 2024-12-05T12:14:07,389 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/3880494075004290aef17a068857dc1f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3880494075004290aef17a068857dc1f 2024-12-05T12:14:07,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/cd4742d29d334f94b4c6c8ee508ab974 is 50, key is test_row_0/B:col10/1733400845657/Put/seqid=0 2024-12-05T12:14:07,399 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/C of dde686338ee027649f1546b042d1ca1b into 3880494075004290aef17a068857dc1f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:07,399 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:07,399 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/C, priority=13, startTime=1733400846510; duration=0sec 2024-12-05T12:14:07,400 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:07,400 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:C 2024-12-05T12:14:07,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741996_1172 (size=12151) 2024-12-05T12:14:07,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400907473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:07,808 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/cd4742d29d334f94b4c6c8ee508ab974 2024-12-05T12:14:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/f70e46a1feeb4b20ad2917acdeeaa359 is 50, key is test_row_0/C:col10/1733400845657/Put/seqid=0 2024-12-05T12:14:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741997_1173 (size=12151) 2024-12-05T12:14:07,824 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/f70e46a1feeb4b20ad2917acdeeaa359 2024-12-05T12:14:07,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/70bef445702745979a2954bea8277de3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/70bef445702745979a2954bea8277de3 2024-12-05T12:14:07,839 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/70bef445702745979a2954bea8277de3, entries=150, sequenceid=181, filesize=30.4 K 2024-12-05T12:14:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/cd4742d29d334f94b4c6c8ee508ab974 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/cd4742d29d334f94b4c6c8ee508ab974 2024-12-05T12:14:07,848 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/cd4742d29d334f94b4c6c8ee508ab974, entries=150, sequenceid=181, filesize=11.9 K 2024-12-05T12:14:07,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/f70e46a1feeb4b20ad2917acdeeaa359 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/f70e46a1feeb4b20ad2917acdeeaa359 2024-12-05T12:14:07,854 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/f70e46a1feeb4b20ad2917acdeeaa359, entries=150, sequenceid=181, filesize=11.9 K 2024-12-05T12:14:07,855 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for dde686338ee027649f1546b042d1ca1b in 1324ms, sequenceid=181, compaction requested=false 2024-12-05T12:14:07,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:07,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:07,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-05T12:14:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-05T12:14:07,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-05T12:14:07,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0900 sec 2024-12-05T12:14:07,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.0950 sec 2024-12-05T12:14:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T12:14:07,913 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:07,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:07,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-05T12:14:07,916 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:07,917 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:07,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:07,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T12:14:07,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:07,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T12:14:07,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:07,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:07,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:07,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:07,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:07,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:07,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053934695d9c934fd5a4875c99229a3cb6_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400847983/Put/seqid=0 2024-12-05T12:14:08,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741998_1174 (size=12304) 2024-12-05T12:14:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T12:14:08,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400908044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:08,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:08,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400908052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:08,054 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:08,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T12:14:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400908073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:08,076 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:08,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400908083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:08,085 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:08,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400908153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T12:14:08,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T12:14:08,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:08,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400908363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:08,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T12:14:08,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:08,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,423 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:08,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053934695d9c934fd5a4875c99229a3cb6_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053934695d9c934fd5a4875c99229a3cb6_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:08,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/004d5ec8388846cb83058fe3ef7e2bf6, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:08,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/004d5ec8388846cb83058fe3ef7e2bf6 is 175, key is test_row_0/A:col10/1733400847983/Put/seqid=0 2024-12-05T12:14:08,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741999_1175 (size=31105) 2024-12-05T12:14:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T12:14:08,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T12:14:08,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:08,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400908672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:08,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T12:14:08,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:08,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/004d5ec8388846cb83058fe3ef7e2bf6 2024-12-05T12:14:08,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T12:14:08,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:08,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:08,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:08,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/29ef646a416d4e3cb87ab1e2c7f24759 is 50, key is test_row_0/B:col10/1733400847983/Put/seqid=0 2024-12-05T12:14:08,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742000_1176 (size=12151) 2024-12-05T12:14:08,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/29ef646a416d4e3cb87ab1e2c7f24759 2024-12-05T12:14:08,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/25c28a47a9394592a6333b295cd5ce3c is 50, key is test_row_0/C:col10/1733400847983/Put/seqid=0 2024-12-05T12:14:08,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742001_1177 (size=12151) 2024-12-05T12:14:09,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T12:14:09,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:09,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:09,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T12:14:09,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T12:14:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:09,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:09,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400909183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:09,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T12:14:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:09,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/25c28a47a9394592a6333b295cd5ce3c 2024-12-05T12:14:09,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/004d5ec8388846cb83058fe3ef7e2bf6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/004d5ec8388846cb83058fe3ef7e2bf6 2024-12-05T12:14:09,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/004d5ec8388846cb83058fe3ef7e2bf6, entries=150, sequenceid=197, filesize=30.4 K 2024-12-05T12:14:09,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/29ef646a416d4e3cb87ab1e2c7f24759 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29ef646a416d4e3cb87ab1e2c7f24759 2024-12-05T12:14:09,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29ef646a416d4e3cb87ab1e2c7f24759, entries=150, sequenceid=197, filesize=11.9 K 2024-12-05T12:14:09,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/25c28a47a9394592a6333b295cd5ce3c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/25c28a47a9394592a6333b295cd5ce3c 2024-12-05T12:14:09,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/25c28a47a9394592a6333b295cd5ce3c, entries=150, sequenceid=197, filesize=11.9 K 2024-12-05T12:14:09,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for dde686338ee027649f1546b042d1ca1b in 1392ms, sequenceid=197, compaction requested=true 2024-12-05T12:14:09,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:09,378 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:09,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:09,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:09,383 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:09,383 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/A is initiating minor compaction (all files) 2024-12-05T12:14:09,383 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/A in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,384 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/ecac83d0bba44e0c876b9486265f5998, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/70bef445702745979a2954bea8277de3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/004d5ec8388846cb83058fe3ef7e2bf6] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=91.4 K 2024-12-05T12:14:09,384 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,384 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/ecac83d0bba44e0c876b9486265f5998, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/70bef445702745979a2954bea8277de3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/004d5ec8388846cb83058fe3ef7e2bf6] 2024-12-05T12:14:09,385 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecac83d0bba44e0c876b9486265f5998, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733400844944 2024-12-05T12:14:09,386 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70bef445702745979a2954bea8277de3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733400845650 2024-12-05T12:14:09,386 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:09,387 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 004d5ec8388846cb83058fe3ef7e2bf6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733400846827 2024-12-05T12:14:09,388 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:09,388 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/B is initiating minor compaction (all files) 2024-12-05T12:14:09,389 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/B in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,389 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/864b3679c3524b8caa0924854bea028a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/cd4742d29d334f94b4c6c8ee508ab974, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29ef646a416d4e3cb87ab1e2c7f24759] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=35.9 K 2024-12-05T12:14:09,389 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 864b3679c3524b8caa0924854bea028a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733400844944 2024-12-05T12:14:09,390 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting cd4742d29d334f94b4c6c8ee508ab974, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733400845650 2024-12-05T12:14:09,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:09,391 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 29ef646a416d4e3cb87ab1e2c7f24759, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733400846827 2024-12-05T12:14:09,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:09,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:09,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:09,405 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:09,425 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:09,426 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/5a51017038124cad9e5faac5ab830b51 is 50, key is test_row_0/B:col10/1733400847983/Put/seqid=0 2024-12-05T12:14:09,436 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120578e63932e6144a10bd4dabd47cf0faa5_dde686338ee027649f1546b042d1ca1b store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:09,439 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120578e63932e6144a10bd4dabd47cf0faa5_dde686338ee027649f1546b042d1ca1b, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:09,439 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120578e63932e6144a10bd4dabd47cf0faa5_dde686338ee027649f1546b042d1ca1b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:09,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742002_1178 (size=12561) 2024-12-05T12:14:09,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T12:14:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,469 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T12:14:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:09,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:09,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:09,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:09,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:09,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:09,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742003_1179 (size=4469) 2024-12-05T12:14:09,474 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/5a51017038124cad9e5faac5ab830b51 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/5a51017038124cad9e5faac5ab830b51 2024-12-05T12:14:09,485 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/B of dde686338ee027649f1546b042d1ca1b into 5a51017038124cad9e5faac5ab830b51(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:09,485 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:09,485 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/B, priority=13, startTime=1733400849386; duration=0sec 2024-12-05T12:14:09,485 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:09,485 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:B 2024-12-05T12:14:09,485 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:09,487 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:09,487 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/C is initiating minor compaction (all files) 2024-12-05T12:14:09,487 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/C in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:09,487 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3880494075004290aef17a068857dc1f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/f70e46a1feeb4b20ad2917acdeeaa359, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/25c28a47a9394592a6333b295cd5ce3c] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=35.9 K 2024-12-05T12:14:09,488 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 3880494075004290aef17a068857dc1f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733400844944 2024-12-05T12:14:09,488 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting f70e46a1feeb4b20ad2917acdeeaa359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733400845650 2024-12-05T12:14:09,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f65a281a98514f98abc1a011001b31e3_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400848029/Put/seqid=0 2024-12-05T12:14:09,492 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 25c28a47a9394592a6333b295cd5ce3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733400846827 2024-12-05T12:14:09,509 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#C#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:09,510 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/a51bbb76e4814c8e93c8366b457c85e4 is 50, key is test_row_0/C:col10/1733400847983/Put/seqid=0 2024-12-05T12:14:09,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742004_1180 (size=12304) 2024-12-05T12:14:09,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:09,554 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f65a281a98514f98abc1a011001b31e3_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f65a281a98514f98abc1a011001b31e3_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:09,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/9b58aae8c16d495bb28e6da0ec2cc90b, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:09,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/9b58aae8c16d495bb28e6da0ec2cc90b is 175, key is test_row_0/A:col10/1733400848029/Put/seqid=0 2024-12-05T12:14:09,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742005_1181 (size=12561) 2024-12-05T12:14:09,574 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/a51bbb76e4814c8e93c8366b457c85e4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a51bbb76e4814c8e93c8366b457c85e4 2024-12-05T12:14:09,584 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/C of dde686338ee027649f1546b042d1ca1b into a51bbb76e4814c8e93c8366b457c85e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:09,584 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:09,584 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/C, priority=13, startTime=1733400849398; duration=0sec 2024-12-05T12:14:09,585 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:09,586 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:C 2024-12-05T12:14:09,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742006_1182 (size=31105) 2024-12-05T12:14:09,591 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/9b58aae8c16d495bb28e6da0ec2cc90b 2024-12-05T12:14:09,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/29aaec7831b746008c98b672c809999c is 50, key is test_row_0/B:col10/1733400848029/Put/seqid=0 2024-12-05T12:14:09,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742007_1183 (size=12151) 2024-12-05T12:14:09,636 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/29aaec7831b746008c98b672c809999c 2024-12-05T12:14:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/29db028e4ada46c38200f82df7c73cc8 is 50, key is test_row_0/C:col10/1733400848029/Put/seqid=0 2024-12-05T12:14:09,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742008_1184 (size=12151) 2024-12-05T12:14:09,873 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#A#compaction#150 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:09,873 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/c6c5f7acbc3744b393dc69a233803171 is 175, key is test_row_0/A:col10/1733400847983/Put/seqid=0 2024-12-05T12:14:09,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742009_1185 (size=31515) 2024-12-05T12:14:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T12:14:10,125 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/29db028e4ada46c38200f82df7c73cc8 2024-12-05T12:14:10,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/9b58aae8c16d495bb28e6da0ec2cc90b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9b58aae8c16d495bb28e6da0ec2cc90b 2024-12-05T12:14:10,140 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9b58aae8c16d495bb28e6da0ec2cc90b, entries=150, sequenceid=216, filesize=30.4 K 2024-12-05T12:14:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/29aaec7831b746008c98b672c809999c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29aaec7831b746008c98b672c809999c 2024-12-05T12:14:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,149 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29aaec7831b746008c98b672c809999c, entries=150, sequenceid=216, filesize=11.9 K 2024-12-05T12:14:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/29db028e4ada46c38200f82df7c73cc8 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/29db028e4ada46c38200f82df7c73cc8 2024-12-05T12:14:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,157 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/29db028e4ada46c38200f82df7c73cc8, entries=150, sequenceid=216, filesize=11.9 K 2024-12-05T12:14:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,158 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for dde686338ee027649f1546b042d1ca1b in 689ms, sequenceid=216, compaction requested=false 2024-12-05T12:14:10,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:10,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:10,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-05T12:14:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-05T12:14:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-05T12:14:10,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2430 sec 2024-12-05T12:14:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.2520 sec 2024-12-05T12:14:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,315 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/c6c5f7acbc3744b393dc69a233803171 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c6c5f7acbc3744b393dc69a233803171 2024-12-05T12:14:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,321 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/A of dde686338ee027649f1546b042d1ca1b into c6c5f7acbc3744b393dc69a233803171(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:10,321 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:10,321 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/A, priority=13, startTime=1733400849378; duration=0sec 2024-12-05T12:14:10,321 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:10,321 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:A 2024-12-05T12:14:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:14:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:10,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:10,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:10,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:10,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:10,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b7c5943ba1e84ac4a57a7ae078bdb5b2_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400850350/Put/seqid=0 2024-12-05T12:14:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742010_1186 (size=22268) 2024-12-05T12:14:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,414 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,419 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b7c5943ba1e84ac4a57a7ae078bdb5b2_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b7c5943ba1e84ac4a57a7ae078bdb5b2_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,421 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/e3b80886ab3b4b279c66c9a7ae942703, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/e3b80886ab3b4b279c66c9a7ae942703 is 175, key is test_row_0/A:col10/1733400850350/Put/seqid=0 2024-12-05T12:14:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:10,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742011_1187 (size=65673) 2024-12-05T12:14:10,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/e3b80886ab3b4b279c66c9a7ae942703 2024-12-05T12:14:10,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/9ba597ceccbf4d6ea02bbd6f6592b47e is 50, key is test_row_0/B:col10/1733400850350/Put/seqid=0 2024-12-05T12:14:10,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400850437, max timestamp=1733400850463Num uniques 859; 2024-12-05T12:14:10,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:14:10,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400850437, max timestamp=1733400850463Num uniques 859; 2024-12-05T12:14:10,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:14:10,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400850437, max timestamp=1733400850463Num uniques 859; 2024-12-05T12:14:10,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:14:10,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742012_1188 (size=12151) 2024-12-05T12:14:10,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/9ba597ceccbf4d6ea02bbd6f6592b47e 2024-12-05T12:14:10,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/5b1271a0d8564c61a31c55e5b8cd96b6 is 50, key is test_row_0/C:col10/1733400850350/Put/seqid=0 2024-12-05T12:14:10,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742013_1189 (size=12151) 2024-12-05T12:14:10,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400910569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:10,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400910683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:10,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400910894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:10,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:10,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:10,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:10,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/5b1271a0d8564c61a31c55e5b8cd96b6 2024-12-05T12:14:10,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/e3b80886ab3b4b279c66c9a7ae942703 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e3b80886ab3b4b279c66c9a7ae942703 2024-12-05T12:14:10,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e3b80886ab3b4b279c66c9a7ae942703, entries=350, sequenceid=229, filesize=64.1 K 2024-12-05T12:14:10,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/9ba597ceccbf4d6ea02bbd6f6592b47e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/9ba597ceccbf4d6ea02bbd6f6592b47e 2024-12-05T12:14:10,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/9ba597ceccbf4d6ea02bbd6f6592b47e, entries=150, sequenceid=229, filesize=11.9 K 2024-12-05T12:14:11,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/5b1271a0d8564c61a31c55e5b8cd96b6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/5b1271a0d8564c61a31c55e5b8cd96b6 2024-12-05T12:14:11,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/5b1271a0d8564c61a31c55e5b8cd96b6, entries=150, sequenceid=229, filesize=11.9 K 2024-12-05T12:14:11,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for dde686338ee027649f1546b042d1ca1b in 662ms, sequenceid=229, compaction requested=true 2024-12-05T12:14:11,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:11,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:11,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:11,013 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:11,013 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:11,015 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:11,015 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/B is initiating minor compaction (all files) 2024-12-05T12:14:11,015 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/B in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:11,015 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/5a51017038124cad9e5faac5ab830b51, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29aaec7831b746008c98b672c809999c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/9ba597ceccbf4d6ea02bbd6f6592b47e] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=36.0 K 2024-12-05T12:14:11,015 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128293 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:11,015 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/A is initiating minor compaction (all files) 2024-12-05T12:14:11,015 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/A in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:11,015 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c6c5f7acbc3744b393dc69a233803171, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9b58aae8c16d495bb28e6da0ec2cc90b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e3b80886ab3b4b279c66c9a7ae942703] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=125.3 K 2024-12-05T12:14:11,016 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:11,016 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c6c5f7acbc3744b393dc69a233803171, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9b58aae8c16d495bb28e6da0ec2cc90b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e3b80886ab3b4b279c66c9a7ae942703] 2024-12-05T12:14:11,016 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6c5f7acbc3744b393dc69a233803171, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733400846827 2024-12-05T12:14:11,017 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a51017038124cad9e5faac5ab830b51, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733400846827 2024-12-05T12:14:11,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:11,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:11,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:11,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:11,017 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b58aae8c16d495bb28e6da0ec2cc90b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733400848029 2024-12-05T12:14:11,017 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 29aaec7831b746008c98b672c809999c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733400848029 2024-12-05T12:14:11,017 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3b80886ab3b4b279c66c9a7ae942703, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733400850216 2024-12-05T12:14:11,017 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba597ceccbf4d6ea02bbd6f6592b47e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733400850337 2024-12-05T12:14:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:11,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-05T12:14:11,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:11,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:11,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:11,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:11,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:11,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:11,040 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#B#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:11,041 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/5d99da7c00794ddaba4fbba645e994ea is 50, key is test_row_0/B:col10/1733400850350/Put/seqid=0 2024-12-05T12:14:11,048 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:11,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058f6ae3cca4914cb29c46b5f15939dacd_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400851019/Put/seqid=0 2024-12-05T12:14:11,059 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412052368cb0df4844a069f64542924f0c946_dde686338ee027649f1546b042d1ca1b store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:11,061 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412052368cb0df4844a069f64542924f0c946_dde686338ee027649f1546b042d1ca1b, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:11,062 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052368cb0df4844a069f64542924f0c946_dde686338ee027649f1546b042d1ca1b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:11,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400911056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:11,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742014_1190 (size=12663) 2024-12-05T12:14:11,096 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/5d99da7c00794ddaba4fbba645e994ea as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/5d99da7c00794ddaba4fbba645e994ea 2024-12-05T12:14:11,107 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/B of dde686338ee027649f1546b042d1ca1b into 5d99da7c00794ddaba4fbba645e994ea(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:11,108 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:11,108 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/B, priority=13, startTime=1733400851013; duration=0sec 2024-12-05T12:14:11,108 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:11,108 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:B 2024-12-05T12:14:11,108 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:11,109 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:11,109 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/C is initiating minor compaction (all files) 2024-12-05T12:14:11,109 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/C in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:11,109 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a51bbb76e4814c8e93c8366b457c85e4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/29db028e4ada46c38200f82df7c73cc8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/5b1271a0d8564c61a31c55e5b8cd96b6] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=36.0 K 2024-12-05T12:14:11,110 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting a51bbb76e4814c8e93c8366b457c85e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733400846827 2024-12-05T12:14:11,110 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 29db028e4ada46c38200f82df7c73cc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733400848029 2024-12-05T12:14:11,112 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b1271a0d8564c61a31c55e5b8cd96b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733400850337 2024-12-05T12:14:11,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742015_1191 (size=14994) 2024-12-05T12:14:11,129 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058f6ae3cca4914cb29c46b5f15939dacd_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058f6ae3cca4914cb29c46b5f15939dacd_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:11,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/fa04b0489c334afb9cc35a2afeadb6a2, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:11,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/fa04b0489c334afb9cc35a2afeadb6a2 is 175, key is test_row_0/A:col10/1733400851019/Put/seqid=0 2024-12-05T12:14:11,149 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:11,149 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/bf5e245735c04f5bb3d3f3913eebfe96 is 50, key is test_row_0/C:col10/1733400850350/Put/seqid=0 2024-12-05T12:14:11,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742016_1192 (size=4469) 2024-12-05T12:14:11,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742017_1193 (size=39949) 2024-12-05T12:14:11,162 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#A#compaction#161 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:11,162 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/fa04b0489c334afb9cc35a2afeadb6a2 2024-12-05T12:14:11,163 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/83c814174e594c6b83f9ba3371171c81 is 175, key is test_row_0/A:col10/1733400850350/Put/seqid=0 2024-12-05T12:14:11,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742018_1194 (size=12663) 2024-12-05T12:14:11,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400911173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:11,187 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/bf5e245735c04f5bb3d3f3913eebfe96 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bf5e245735c04f5bb3d3f3913eebfe96 2024-12-05T12:14:11,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/637acd565713477b9e88b3533c13eba1 is 50, key is test_row_0/B:col10/1733400851019/Put/seqid=0 2024-12-05T12:14:11,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742019_1195 (size=31617) 2024-12-05T12:14:11,198 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/83c814174e594c6b83f9ba3371171c81 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/83c814174e594c6b83f9ba3371171c81 2024-12-05T12:14:11,199 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/C of dde686338ee027649f1546b042d1ca1b into bf5e245735c04f5bb3d3f3913eebfe96(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:11,199 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:11,199 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/C, priority=13, startTime=1733400851017; duration=0sec 2024-12-05T12:14:11,199 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:11,199 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:C 2024-12-05T12:14:11,209 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/A of dde686338ee027649f1546b042d1ca1b into 83c814174e594c6b83f9ba3371171c81(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:11,209 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:11,209 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/A, priority=13, startTime=1733400851013; duration=0sec 2024-12-05T12:14:11,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400911203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:11,210 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:11,210 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:A 2024-12-05T12:14:11,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742020_1196 (size=12301) 2024-12-05T12:14:11,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/637acd565713477b9e88b3533c13eba1 2024-12-05T12:14:11,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/3295a9da8e9f4b518b64a5b8e3c1cbfc is 50, key is test_row_0/C:col10/1733400851019/Put/seqid=0 2024-12-05T12:14:11,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742021_1197 (size=12301) 2024-12-05T12:14:11,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/3295a9da8e9f4b518b64a5b8e3c1cbfc 2024-12-05T12:14:11,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/fa04b0489c334afb9cc35a2afeadb6a2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/fa04b0489c334afb9cc35a2afeadb6a2 2024-12-05T12:14:11,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/fa04b0489c334afb9cc35a2afeadb6a2, entries=200, sequenceid=266, filesize=39.0 K 2024-12-05T12:14:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/637acd565713477b9e88b3533c13eba1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/637acd565713477b9e88b3533c13eba1 2024-12-05T12:14:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/637acd565713477b9e88b3533c13eba1, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T12:14:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/3295a9da8e9f4b518b64a5b8e3c1cbfc as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3295a9da8e9f4b518b64a5b8e3c1cbfc 2024-12-05T12:14:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3295a9da8e9f4b518b64a5b8e3c1cbfc, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T12:14:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for dde686338ee027649f1546b042d1ca1b in 282ms, sequenceid=266, compaction requested=false 2024-12-05T12:14:11,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:14:11,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:11,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:11,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:11,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:11,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:11,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057d32506d266b44e1995b2c1c5f84d26c_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400851387/Put/seqid=0 2024-12-05T12:14:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742022_1198 (size=12454) 2024-12-05T12:14:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400851457, max timestamp=1733400851494Num uniques 859; 2024-12-05T12:14:11,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400851457, max timestamp=1733400851494Num uniques 859; 2024-12-05T12:14:11,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:14:11,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:14:11,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400851457, max timestamp=1733400851494Num uniques 859; 2024-12-05T12:14:11,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:14:11,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400911581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400911693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:11,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:11,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400911736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:11,836 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:11,848 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057d32506d266b44e1995b2c1c5f84d26c_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057d32506d266b44e1995b2c1c5f84d26c_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:11,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/69db3a8806f64259a39282c126067a90, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:11,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/69db3a8806f64259a39282c126067a90 is 175, key is test_row_0/A:col10/1733400851387/Put/seqid=0 2024-12-05T12:14:11,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400911907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:11,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742023_1199 (size=31251) 2024-12-05T12:14:11,915 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/69db3a8806f64259a39282c126067a90 2024-12-05T12:14:11,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/a55303bf3d694db881f25ebfffdc65ef is 50, key is test_row_0/B:col10/1733400851387/Put/seqid=0 2024-12-05T12:14:11,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742024_1200 (size=9857) 2024-12-05T12:14:12,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48826 deadline: 1733400912064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:12,066 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T12:14:12,073 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:12,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-05T12:14:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T12:14:12,079 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:12,081 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:12,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:12,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1733400912099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:12,102 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:12,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48882 deadline: 1733400912113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:12,116 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T12:14:12,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400912228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:12,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T12:14:12,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:12,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/a55303bf3d694db881f25ebfffdc65ef 2024-12-05T12:14:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T12:14:12,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/350daa49690e4332abcb1af0682107ad is 50, key is test_row_0/C:col10/1733400851387/Put/seqid=0 2024-12-05T12:14:12,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T12:14:12,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742025_1201 (size=9857) 2024-12-05T12:14:12,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T12:14:12,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:12,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T12:14:12,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T12:14:12,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:12,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400912743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:12,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400912753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:12,760 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:12,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/350daa49690e4332abcb1af0682107ad 2024-12-05T12:14:12,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/69db3a8806f64259a39282c126067a90 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/69db3a8806f64259a39282c126067a90 2024-12-05T12:14:12,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/69db3a8806f64259a39282c126067a90, entries=150, sequenceid=279, filesize=30.5 K 2024-12-05T12:14:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/a55303bf3d694db881f25ebfffdc65ef as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/a55303bf3d694db881f25ebfffdc65ef 2024-12-05T12:14:12,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T12:14:12,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:12,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:12,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/a55303bf3d694db881f25ebfffdc65ef, entries=100, sequenceid=279, filesize=9.6 K 2024-12-05T12:14:12,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/350daa49690e4332abcb1af0682107ad as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/350daa49690e4332abcb1af0682107ad 2024-12-05T12:14:12,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/350daa49690e4332abcb1af0682107ad, entries=100, sequenceid=279, filesize=9.6 K 2024-12-05T12:14:12,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for dde686338ee027649f1546b042d1ca1b in 1498ms, sequenceid=279, compaction requested=true 2024-12-05T12:14:12,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:12,888 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:12,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:12,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:12,889 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:12,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:12,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:12,889 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102817 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:12,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:12,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:12,889 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/A is initiating minor compaction (all files) 2024-12-05T12:14:12,889 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/A in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,890 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/83c814174e594c6b83f9ba3371171c81, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/fa04b0489c334afb9cc35a2afeadb6a2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/69db3a8806f64259a39282c126067a90] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=100.4 K 2024-12-05T12:14:12,890 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,890 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/83c814174e594c6b83f9ba3371171c81, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/fa04b0489c334afb9cc35a2afeadb6a2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/69db3a8806f64259a39282c126067a90] 2024-12-05T12:14:12,890 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83c814174e594c6b83f9ba3371171c81, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733400850337 2024-12-05T12:14:12,891 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:12,891 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/B is initiating minor compaction (all files) 2024-12-05T12:14:12,891 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa04b0489c334afb9cc35a2afeadb6a2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733400850517 2024-12-05T12:14:12,891 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/B in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:12,891 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/5d99da7c00794ddaba4fbba645e994ea, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/637acd565713477b9e88b3533c13eba1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/a55303bf3d694db881f25ebfffdc65ef] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=34.0 K 2024-12-05T12:14:12,892 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69db3a8806f64259a39282c126067a90, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733400851046 2024-12-05T12:14:12,892 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d99da7c00794ddaba4fbba645e994ea, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733400850337 2024-12-05T12:14:12,892 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 637acd565713477b9e88b3533c13eba1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733400850517 2024-12-05T12:14:12,893 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting a55303bf3d694db881f25ebfffdc65ef, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733400851046 2024-12-05T12:14:12,901 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:12,908 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:12,909 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/50a0ddd0d0e04da489bf3fae3eb5e6ee is 50, key is test_row_0/B:col10/1733400851387/Put/seqid=0 2024-12-05T12:14:12,921 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205cd25297bc2aa48adb3dc9c646b5b694b_dde686338ee027649f1546b042d1ca1b store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:12,923 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205cd25297bc2aa48adb3dc9c646b5b694b_dde686338ee027649f1546b042d1ca1b, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:12,924 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cd25297bc2aa48adb3dc9c646b5b694b_dde686338ee027649f1546b042d1ca1b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:12,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742026_1202 (size=12915) 2024-12-05T12:14:12,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742027_1203 (size=4469) 2024-12-05T12:14:13,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T12:14:13,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:13,031 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-05T12:14:13,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:13,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c6ce934f267c48c2b768baa4f2ff7e78_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400851572/Put/seqid=0 2024-12-05T12:14:13,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742028_1204 (size=12454) 2024-12-05T12:14:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T12:14:13,383 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#A#compaction#168 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:13,384 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/a5fbca39c080475f95608d4f9f80dfca is 175, key is test_row_0/A:col10/1733400851387/Put/seqid=0 2024-12-05T12:14:13,394 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/50a0ddd0d0e04da489bf3fae3eb5e6ee as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/50a0ddd0d0e04da489bf3fae3eb5e6ee 2024-12-05T12:14:13,402 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/B of dde686338ee027649f1546b042d1ca1b into 50a0ddd0d0e04da489bf3fae3eb5e6ee(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:13,402 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:13,402 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/B, priority=13, startTime=1733400852889; duration=0sec 2024-12-05T12:14:13,403 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:13,403 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:B 2024-12-05T12:14:13,403 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:13,404 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:13,405 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/C is initiating minor compaction (all files) 2024-12-05T12:14:13,405 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/C in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:13,405 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bf5e245735c04f5bb3d3f3913eebfe96, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3295a9da8e9f4b518b64a5b8e3c1cbfc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/350daa49690e4332abcb1af0682107ad] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=34.0 K 2024-12-05T12:14:13,406 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5e245735c04f5bb3d3f3913eebfe96, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733400850337 2024-12-05T12:14:13,408 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 3295a9da8e9f4b518b64a5b8e3c1cbfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733400850517 2024-12-05T12:14:13,409 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 350daa49690e4332abcb1af0682107ad, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733400851046 2024-12-05T12:14:13,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742029_1205 (size=31976) 2024-12-05T12:14:13,426 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:13,427 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/84b9c7adc8034e7baf4d656366621e86 is 50, key is test_row_0/C:col10/1733400851387/Put/seqid=0 2024-12-05T12:14:13,430 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/a5fbca39c080475f95608d4f9f80dfca as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/a5fbca39c080475f95608d4f9f80dfca 2024-12-05T12:14:13,446 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/A of dde686338ee027649f1546b042d1ca1b into a5fbca39c080475f95608d4f9f80dfca(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:13,446 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:13,446 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/A, priority=13, startTime=1733400852887; duration=0sec 2024-12-05T12:14:13,455 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:13,455 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:A 2024-12-05T12:14:13,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742030_1206 (size=12915) 2024-12-05T12:14:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:13,517 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/84b9c7adc8034e7baf4d656366621e86 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/84b9c7adc8034e7baf4d656366621e86 2024-12-05T12:14:13,525 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c6ce934f267c48c2b768baa4f2ff7e78_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c6ce934f267c48c2b768baa4f2ff7e78_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:13,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/33ba843be340425d8ccf6b777863a083, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:13,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/33ba843be340425d8ccf6b777863a083 is 175, key is test_row_0/A:col10/1733400851572/Put/seqid=0 2024-12-05T12:14:13,552 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/C of dde686338ee027649f1546b042d1ca1b into 84b9c7adc8034e7baf4d656366621e86(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:13,552 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:13,552 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/C, priority=13, startTime=1733400852889; duration=0sec 2024-12-05T12:14:13,552 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:13,552 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:C 2024-12-05T12:14:13,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742031_1207 (size=31255) 2024-12-05T12:14:13,581 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/33ba843be340425d8ccf6b777863a083 2024-12-05T12:14:13,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b70fe07e2d7e4d4bbf64c6c6d05ab167 is 50, key is test_row_0/B:col10/1733400851572/Put/seqid=0 2024-12-05T12:14:13,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742032_1208 (size=12301) 2024-12-05T12:14:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:13,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:13,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400913795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400913902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:13,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:13,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:13,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:14,069 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b70fe07e2d7e4d4bbf64c6c6d05ab167 2024-12-05T12:14:14,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/b849a62b3eb74673a4bbe2f94d25fa8f is 50, key is test_row_0/C:col10/1733400851572/Put/seqid=0 2024-12-05T12:14:14,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400914134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:14,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742033_1209 (size=12301) 2024-12-05T12:14:14,142 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/b849a62b3eb74673a4bbe2f94d25fa8f 2024-12-05T12:14:14,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/33ba843be340425d8ccf6b777863a083 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/33ba843be340425d8ccf6b777863a083 2024-12-05T12:14:14,187 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/33ba843be340425d8ccf6b777863a083, entries=150, sequenceid=314, filesize=30.5 K 2024-12-05T12:14:14,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/b70fe07e2d7e4d4bbf64c6c6d05ab167 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b70fe07e2d7e4d4bbf64c6c6d05ab167 2024-12-05T12:14:14,224 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b70fe07e2d7e4d4bbf64c6c6d05ab167, entries=150, sequenceid=314, filesize=12.0 K 2024-12-05T12:14:14,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/b849a62b3eb74673a4bbe2f94d25fa8f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/b849a62b3eb74673a4bbe2f94d25fa8f 2024-12-05T12:14:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T12:14:14,246 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/b849a62b3eb74673a4bbe2f94d25fa8f, entries=150, sequenceid=314, filesize=12.0 K 2024-12-05T12:14:14,263 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for dde686338ee027649f1546b042d1ca1b in 1231ms, sequenceid=314, compaction requested=false 2024-12-05T12:14:14,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:14,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:14,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-05T12:14:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-05T12:14:14,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-05T12:14:14,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1840 sec 2024-12-05T12:14:14,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.1940 sec 2024-12-05T12:14:14,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T12:14:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:14,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050a5ec60d3c794ecf9a87399d531faf00_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400853792/Put/seqid=0 2024-12-05T12:14:14,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742034_1210 (size=12454) 2024-12-05T12:14:14,515 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T12:14:14,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400854505, max timestamp=1733400854556Num uniques 859; 2024-12-05T12:14:14,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:14:14,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400854505, max timestamp=1733400854556Num uniques 859; 2024-12-05T12:14:14,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:14,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400854505, max timestamp=1733400854556Num uniques 859; 2024-12-05T12:14:14,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:14,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400914675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:14,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400914775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:14,784 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in dde686338ee027649f1546b042d1ca1b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:14,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400914795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:14,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:14,914 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050a5ec60d3c794ecf9a87399d531faf00_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050a5ec60d3c794ecf9a87399d531faf00_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:14,916 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/7a6b704720c442bba37371431e990194, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:14,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/7a6b704720c442bba37371431e990194 is 175, key is test_row_0/A:col10/1733400853792/Put/seqid=0 2024-12-05T12:14:14,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742035_1211 (size=31255) 2024-12-05T12:14:14,943 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/7a6b704720c442bba37371431e990194 2024-12-05T12:14:14,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/ae371408532346abaae8c3809bbef411 is 50, key is test_row_0/B:col10/1733400853792/Put/seqid=0 2024-12-05T12:14:14,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742036_1212 (size=12301) 2024-12-05T12:14:14,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/ae371408532346abaae8c3809bbef411 2024-12-05T12:14:15,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400915002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:15,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/d905228a16684386a272feb2bc86a0a0 is 50, key is test_row_0/C:col10/1733400853792/Put/seqid=0 2024-12-05T12:14:15,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742037_1213 (size=12301) 2024-12-05T12:14:15,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/d905228a16684386a272feb2bc86a0a0 2024-12-05T12:14:15,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/7a6b704720c442bba37371431e990194 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/7a6b704720c442bba37371431e990194 2024-12-05T12:14:15,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/7a6b704720c442bba37371431e990194, entries=150, sequenceid=328, filesize=30.5 K 2024-12-05T12:14:15,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/ae371408532346abaae8c3809bbef411 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/ae371408532346abaae8c3809bbef411 2024-12-05T12:14:15,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/ae371408532346abaae8c3809bbef411, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T12:14:15,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/d905228a16684386a272feb2bc86a0a0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/d905228a16684386a272feb2bc86a0a0 2024-12-05T12:14:15,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/d905228a16684386a272feb2bc86a0a0, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T12:14:15,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for dde686338ee027649f1546b042d1ca1b in 672ms, sequenceid=328, compaction requested=true 2024-12-05T12:14:15,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:15,118 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:15,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:15,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:15,120 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:15,121 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94486 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:15,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:15,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:15,121 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/A is initiating minor compaction (all files) 2024-12-05T12:14:15,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:15,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:15,121 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/A in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:15,121 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/a5fbca39c080475f95608d4f9f80dfca, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/33ba843be340425d8ccf6b777863a083, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/7a6b704720c442bba37371431e990194] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=92.3 K 2024-12-05T12:14:15,121 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:15,121 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/a5fbca39c080475f95608d4f9f80dfca, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/33ba843be340425d8ccf6b777863a083, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/7a6b704720c442bba37371431e990194] 2024-12-05T12:14:15,122 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5fbca39c080475f95608d4f9f80dfca, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733400850559 2024-12-05T12:14:15,122 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:15,122 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/B is initiating minor compaction (all files) 2024-12-05T12:14:15,122 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/B in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:15,123 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/50a0ddd0d0e04da489bf3fae3eb5e6ee, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b70fe07e2d7e4d4bbf64c6c6d05ab167, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/ae371408532346abaae8c3809bbef411] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=36.6 K 2024-12-05T12:14:15,123 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33ba843be340425d8ccf6b777863a083, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733400851540 2024-12-05T12:14:15,123 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 50a0ddd0d0e04da489bf3fae3eb5e6ee, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733400850559 2024-12-05T12:14:15,124 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a6b704720c442bba37371431e990194, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733400853767 2024-12-05T12:14:15,124 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b70fe07e2d7e4d4bbf64c6c6d05ab167, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733400851540 2024-12-05T12:14:15,125 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting ae371408532346abaae8c3809bbef411, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733400853767 2024-12-05T12:14:15,132 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:15,141 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:15,142 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/672ed03175d04b478edcdbabf71e5739 is 50, key is test_row_0/B:col10/1733400853792/Put/seqid=0 2024-12-05T12:14:15,155 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412058266627d458b43d6b49be9c3900714e7_dde686338ee027649f1546b042d1ca1b store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:15,158 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412058266627d458b43d6b49be9c3900714e7_dde686338ee027649f1546b042d1ca1b, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:15,159 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058266627d458b43d6b49be9c3900714e7_dde686338ee027649f1546b042d1ca1b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:15,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742038_1214 (size=13017) 2024-12-05T12:14:15,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742039_1215 (size=4469) 2024-12-05T12:14:15,187 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/672ed03175d04b478edcdbabf71e5739 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/672ed03175d04b478edcdbabf71e5739 2024-12-05T12:14:15,188 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#A#compaction#177 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:15,189 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/029c942a9a7d4717882f19ee476cdb51 is 175, key is test_row_0/A:col10/1733400853792/Put/seqid=0 2024-12-05T12:14:15,197 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/B of dde686338ee027649f1546b042d1ca1b into 672ed03175d04b478edcdbabf71e5739(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:15,197 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:15,197 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/B, priority=13, startTime=1733400855120; duration=0sec 2024-12-05T12:14:15,197 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:15,197 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:B 2024-12-05T12:14:15,197 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:15,199 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:15,199 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/C is initiating minor compaction (all files) 2024-12-05T12:14:15,199 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/C in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:15,199 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/84b9c7adc8034e7baf4d656366621e86, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/b849a62b3eb74673a4bbe2f94d25fa8f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/d905228a16684386a272feb2bc86a0a0] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=36.6 K 2024-12-05T12:14:15,199 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b9c7adc8034e7baf4d656366621e86, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733400850559 2024-12-05T12:14:15,201 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b849a62b3eb74673a4bbe2f94d25fa8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733400851540 2024-12-05T12:14:15,202 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d905228a16684386a272feb2bc86a0a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733400853767 2024-12-05T12:14:15,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742040_1216 (size=31971) 2024-12-05T12:14:15,217 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/029c942a9a7d4717882f19ee476cdb51 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/029c942a9a7d4717882f19ee476cdb51 2024-12-05T12:14:15,225 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/A of dde686338ee027649f1546b042d1ca1b into 029c942a9a7d4717882f19ee476cdb51(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:15,225 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:15,225 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/A, priority=13, startTime=1733400855118; duration=0sec 2024-12-05T12:14:15,225 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:15,225 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:A 2024-12-05T12:14:15,233 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:15,233 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/775c61c3cc804002bfe1805935fa9c7e is 50, key is test_row_0/C:col10/1733400853792/Put/seqid=0 2024-12-05T12:14:15,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742041_1217 (size=13017) 2024-12-05T12:14:15,277 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/775c61c3cc804002bfe1805935fa9c7e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/775c61c3cc804002bfe1805935fa9c7e 2024-12-05T12:14:15,283 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/C of dde686338ee027649f1546b042d1ca1b into 775c61c3cc804002bfe1805935fa9c7e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:15,283 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:15,283 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/C, priority=13, startTime=1733400855121; duration=0sec 2024-12-05T12:14:15,283 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:15,283 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:C 2024-12-05T12:14:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:15,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-12-05T12:14:15,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:15,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:15,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:15,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:15,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:15,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:15,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d981c1c707014130b09c6e19600e26e5_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400854664/Put/seqid=0 2024-12-05T12:14:15,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400915367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:15,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742042_1218 (size=14994) 2024-12-05T12:14:15,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:15,387 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d981c1c707014130b09c6e19600e26e5_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d981c1c707014130b09c6e19600e26e5_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:15,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/5742fb9780d74a2288a39a4beeeb7cb1, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:15,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/5742fb9780d74a2288a39a4beeeb7cb1 is 175, key is test_row_0/A:col10/1733400854664/Put/seqid=0 2024-12-05T12:14:15,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742043_1219 (size=39949) 2024-12-05T12:14:15,415 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/5742fb9780d74a2288a39a4beeeb7cb1 2024-12-05T12:14:15,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/00c525c81fa040ae9758995d36dbdb80 is 50, key is test_row_0/B:col10/1733400854664/Put/seqid=0 2024-12-05T12:14:15,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742044_1220 (size=12301) 2024-12-05T12:14:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400915472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:15,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400915683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:15,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/00c525c81fa040ae9758995d36dbdb80 2024-12-05T12:14:15,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/60e99cbb4fad455aa290d41857976183 is 50, key is test_row_0/C:col10/1733400854664/Put/seqid=0 2024-12-05T12:14:15,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742045_1221 (size=12301) 2024-12-05T12:14:15,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/60e99cbb4fad455aa290d41857976183 2024-12-05T12:14:15,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/5742fb9780d74a2288a39a4beeeb7cb1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5742fb9780d74a2288a39a4beeeb7cb1 2024-12-05T12:14:15,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5742fb9780d74a2288a39a4beeeb7cb1, entries=200, sequenceid=368, filesize=39.0 K 2024-12-05T12:14:15,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/00c525c81fa040ae9758995d36dbdb80 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/00c525c81fa040ae9758995d36dbdb80 2024-12-05T12:14:15,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/00c525c81fa040ae9758995d36dbdb80, entries=150, sequenceid=368, filesize=12.0 K 2024-12-05T12:14:15,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/60e99cbb4fad455aa290d41857976183 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/60e99cbb4fad455aa290d41857976183 2024-12-05T12:14:15,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/60e99cbb4fad455aa290d41857976183, entries=150, sequenceid=368, filesize=12.0 K 2024-12-05T12:14:15,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for dde686338ee027649f1546b042d1ca1b in 680ms, sequenceid=368, compaction requested=false 2024-12-05T12:14:15,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:16,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T12:14:16,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:16,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:16,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:16,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:16,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:16,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:16,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050575d94d481a414a84791d12ccd32243_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400856000/Put/seqid=0 2024-12-05T12:14:16,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742046_1222 (size=14994) 2024-12-05T12:14:16,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400856102, max timestamp=1733400856216Num uniques 859; 2024-12-05T12:14:16,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:16,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400856102, max timestamp=1733400856216Num uniques 859; 2024-12-05T12:14:16,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:16,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400856102, max timestamp=1733400856216Num uniques 859; 2024-12-05T12:14:16,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T12:14:16,264 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:16,267 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-05T12:14:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T12:14:16,275 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:16,283 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:16,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:16,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400916356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T12:14:16,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T12:14:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:16,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400916473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:16,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:16,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:16,545 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:16,555 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050575d94d481a414a84791d12ccd32243_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050575d94d481a414a84791d12ccd32243_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:16,556 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/3ce5d3b9c4584be2bc6d5a4b36fdce64, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:16,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/3ce5d3b9c4584be2bc6d5a4b36fdce64 is 175, key is test_row_0/A:col10/1733400856000/Put/seqid=0 2024-12-05T12:14:16,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742047_1223 (size=39949) 2024-12-05T12:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T12:14:16,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T12:14:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:16,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:16,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400916683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:16,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T12:14:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:16,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T12:14:16,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T12:14:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:16,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:16,976 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/3ce5d3b9c4584be2bc6d5a4b36fdce64 2024-12-05T12:14:16,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400916994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:16,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/38022f947f45408997d243dde6be4177 is 50, key is test_row_0/B:col10/1733400856000/Put/seqid=0 2024-12-05T12:14:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:17,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742048_1224 (size=12301) 2024-12-05T12:14:17,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/38022f947f45408997d243dde6be4177 2024-12-05T12:14:17,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/df7f3a919e1c47aea60d357b20d54482 is 50, key is test_row_0/C:col10/1733400856000/Put/seqid=0 2024-12-05T12:14:17,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T12:14:17,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:17,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:17,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:17,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:17,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:17,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742049_1225 (size=12301) 2024-12-05T12:14:17,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/df7f3a919e1c47aea60d357b20d54482 2024-12-05T12:14:17,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/3ce5d3b9c4584be2bc6d5a4b36fdce64 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3ce5d3b9c4584be2bc6d5a4b36fdce64 2024-12-05T12:14:17,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3ce5d3b9c4584be2bc6d5a4b36fdce64, entries=200, sequenceid=379, filesize=39.0 K 2024-12-05T12:14:17,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/38022f947f45408997d243dde6be4177 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/38022f947f45408997d243dde6be4177 2024-12-05T12:14:17,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/38022f947f45408997d243dde6be4177, entries=150, sequenceid=379, filesize=12.0 K 2024-12-05T12:14:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/df7f3a919e1c47aea60d357b20d54482 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/df7f3a919e1c47aea60d357b20d54482 2024-12-05T12:14:17,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/df7f3a919e1c47aea60d357b20d54482, entries=150, sequenceid=379, filesize=12.0 K 2024-12-05T12:14:17,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dde686338ee027649f1546b042d1ca1b in 1110ms, sequenceid=379, compaction requested=true 2024-12-05T12:14:17,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:17,111 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:17,112 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:17,113 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:17,113 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/A is initiating minor compaction (all files) 2024-12-05T12:14:17,114 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/A in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:17,114 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/029c942a9a7d4717882f19ee476cdb51, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5742fb9780d74a2288a39a4beeeb7cb1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3ce5d3b9c4584be2bc6d5a4b36fdce64] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=109.2 K 2024-12-05T12:14:17,114 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:17,114 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/029c942a9a7d4717882f19ee476cdb51, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5742fb9780d74a2288a39a4beeeb7cb1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3ce5d3b9c4584be2bc6d5a4b36fdce64] 2024-12-05T12:14:17,114 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:17,114 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/B is initiating minor compaction (all files) 2024-12-05T12:14:17,114 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/B in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:17,114 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/672ed03175d04b478edcdbabf71e5739, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/00c525c81fa040ae9758995d36dbdb80, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/38022f947f45408997d243dde6be4177] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=36.7 K 2024-12-05T12:14:17,115 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 029c942a9a7d4717882f19ee476cdb51, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733400853767 2024-12-05T12:14:17,115 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 672ed03175d04b478edcdbabf71e5739, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733400853767 2024-12-05T12:14:17,115 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5742fb9780d74a2288a39a4beeeb7cb1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733400854656 2024-12-05T12:14:17,115 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 00c525c81fa040ae9758995d36dbdb80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733400854664 2024-12-05T12:14:17,116 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 38022f947f45408997d243dde6be4177, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733400855345 2024-12-05T12:14:17,116 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ce5d3b9c4584be2bc6d5a4b36fdce64, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733400855325 2024-12-05T12:14:17,131 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:17,135 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:17,136 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/7511a51940ea41928b6e5cc661fde2a5 is 50, key is test_row_0/B:col10/1733400856000/Put/seqid=0 2024-12-05T12:14:17,140 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205e8554a95e5ae444e9fdac92df9fa02ed_dde686338ee027649f1546b042d1ca1b store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:17,142 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205e8554a95e5ae444e9fdac92df9fa02ed_dde686338ee027649f1546b042d1ca1b, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:17,142 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e8554a95e5ae444e9fdac92df9fa02ed_dde686338ee027649f1546b042d1ca1b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:17,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742050_1226 (size=4469) 2024-12-05T12:14:17,171 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#A#compaction#186 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:17,172 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/66151c266b0042019deda58d67dc9b10 is 175, key is test_row_0/A:col10/1733400856000/Put/seqid=0 2024-12-05T12:14:17,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742051_1227 (size=13119) 2024-12-05T12:14:17,197 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/7511a51940ea41928b6e5cc661fde2a5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/7511a51940ea41928b6e5cc661fde2a5 2024-12-05T12:14:17,209 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/B of dde686338ee027649f1546b042d1ca1b into 7511a51940ea41928b6e5cc661fde2a5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:17,210 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:17,210 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/B, priority=13, startTime=1733400857112; duration=0sec 2024-12-05T12:14:17,210 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:17,210 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:B 2024-12-05T12:14:17,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742052_1228 (size=32073) 2024-12-05T12:14:17,210 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:17,211 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:17,211 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/C is initiating minor compaction (all files) 2024-12-05T12:14:17,211 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/C in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:17,211 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/775c61c3cc804002bfe1805935fa9c7e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/60e99cbb4fad455aa290d41857976183, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/df7f3a919e1c47aea60d357b20d54482] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=36.7 K 2024-12-05T12:14:17,212 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 775c61c3cc804002bfe1805935fa9c7e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733400853767 2024-12-05T12:14:17,212 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e99cbb4fad455aa290d41857976183, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733400854664 2024-12-05T12:14:17,213 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting df7f3a919e1c47aea60d357b20d54482, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733400855345 2024-12-05T12:14:17,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T12:14:17,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:17,226 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-05T12:14:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:17,227 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/66151c266b0042019deda58d67dc9b10 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/66151c266b0042019deda58d67dc9b10 2024-12-05T12:14:17,234 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/A of dde686338ee027649f1546b042d1ca1b into 66151c266b0042019deda58d67dc9b10(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:17,234 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:17,234 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/A, priority=13, startTime=1733400857111; duration=0sec 2024-12-05T12:14:17,234 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:17,234 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:A 2024-12-05T12:14:17,239 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:17,240 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/8f49d1b7ff41465997c3bb8a398cf738 is 50, key is test_row_0/C:col10/1733400856000/Put/seqid=0 2024-12-05T12:14:17,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b543fbb975ac468b847804eb174144d8_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400856300/Put/seqid=0 2024-12-05T12:14:17,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742053_1229 (size=13119) 2024-12-05T12:14:17,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742054_1230 (size=12454) 2024-12-05T12:14:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T12:14:17,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:17,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400917578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:17,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:17,699 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/8f49d1b7ff41465997c3bb8a398cf738 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8f49d1b7ff41465997c3bb8a398cf738 2024-12-05T12:14:17,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400917695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:17,710 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b543fbb975ac468b847804eb174144d8_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b543fbb975ac468b847804eb174144d8_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:17,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/f56697dc7bcb4053a0110eaae31a81cd, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:17,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/f56697dc7bcb4053a0110eaae31a81cd is 175, key is test_row_0/A:col10/1733400856300/Put/seqid=0 2024-12-05T12:14:17,730 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde686338ee027649f1546b042d1ca1b/C of dde686338ee027649f1546b042d1ca1b into 8f49d1b7ff41465997c3bb8a398cf738(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:17,730 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:17,730 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/C, priority=13, startTime=1733400857112; duration=0sec 2024-12-05T12:14:17,730 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:17,731 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:C 2024-12-05T12:14:17,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742055_1231 (size=31255) 2024-12-05T12:14:17,743 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/f56697dc7bcb4053a0110eaae31a81cd 2024-12-05T12:14:17,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/6fea3d8297e14601999a7752833435f7 is 50, key is test_row_0/B:col10/1733400856300/Put/seqid=0 2024-12-05T12:14:17,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742056_1232 (size=12301) 2024-12-05T12:14:17,795 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/6fea3d8297e14601999a7752833435f7 2024-12-05T12:14:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/80cba62f513146009cc4d6bdf52daadc is 50, key is test_row_0/C:col10/1733400856300/Put/seqid=0 2024-12-05T12:14:17,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742057_1233 (size=12301) 2024-12-05T12:14:17,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400917912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:18,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400918223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:18,277 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/80cba62f513146009cc4d6bdf52daadc 2024-12-05T12:14:18,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/f56697dc7bcb4053a0110eaae31a81cd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/f56697dc7bcb4053a0110eaae31a81cd 2024-12-05T12:14:18,304 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/f56697dc7bcb4053a0110eaae31a81cd, entries=150, sequenceid=416, filesize=30.5 K 2024-12-05T12:14:18,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/6fea3d8297e14601999a7752833435f7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/6fea3d8297e14601999a7752833435f7 2024-12-05T12:14:18,313 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/6fea3d8297e14601999a7752833435f7, entries=150, sequenceid=416, filesize=12.0 K 2024-12-05T12:14:18,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/80cba62f513146009cc4d6bdf52daadc as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/80cba62f513146009cc4d6bdf52daadc 2024-12-05T12:14:18,339 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/80cba62f513146009cc4d6bdf52daadc, entries=150, sequenceid=416, filesize=12.0 K 2024-12-05T12:14:18,341 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for dde686338ee027649f1546b042d1ca1b in 1114ms, sequenceid=416, compaction requested=false 2024-12-05T12:14:18,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:18,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:18,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-05T12:14:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-05T12:14:18,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-05T12:14:18,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0590 sec 2024-12-05T12:14:18,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.0780 sec 2024-12-05T12:14:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T12:14:18,423 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:18,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-05T12:14:18,426 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T12:14:18,427 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:18,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T12:14:18,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T12:14:18,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:18,583 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T12:14:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:18,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205058b373c445748a79f07318f7f2cde51_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400857562/Put/seqid=0 2024-12-05T12:14:18,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742058_1234 (size=12454) 2024-12-05T12:14:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:18,659 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205058b373c445748a79f07318f7f2cde51_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205058b373c445748a79f07318f7f2cde51_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:18,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/716b506a450b42e1b23d509345948133, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:18,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/716b506a450b42e1b23d509345948133 is 175, key is test_row_0/A:col10/1733400857562/Put/seqid=0 2024-12-05T12:14:18,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742059_1235 (size=31255) 2024-12-05T12:14:18,708 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/716b506a450b42e1b23d509345948133 2024-12-05T12:14:18,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/470725ded1584a3fa3d0b79c67057d7b is 50, key is test_row_0/B:col10/1733400857562/Put/seqid=0 2024-12-05T12:14:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T12:14:18,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742060_1236 (size=12301) 2024-12-05T12:14:18,754 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/470725ded1584a3fa3d0b79c67057d7b 2024-12-05T12:14:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:18,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. as already flushing 2024-12-05T12:14:18,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/ee47ebea4a584eed973d4d83a0105223 is 50, key is test_row_0/C:col10/1733400857562/Put/seqid=0 2024-12-05T12:14:18,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742061_1237 (size=12301) 2024-12-05T12:14:18,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400858838, max timestamp=1733400858860Num uniques 859; 2024-12-05T12:14:18,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:18,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400858838, max timestamp=1733400858860Num uniques 859; 2024-12-05T12:14:18,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400858838, max timestamp=1733400858860Num uniques 859; 2024-12-05T12:14:18,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:18,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:18,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400918932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:18,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400918932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:19,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400919043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:19,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400919043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T12:14:19,216 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/ee47ebea4a584eed973d4d83a0105223 2024-12-05T12:14:19,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/716b506a450b42e1b23d509345948133 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/716b506a450b42e1b23d509345948133 2024-12-05T12:14:19,227 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/716b506a450b42e1b23d509345948133, entries=150, sequenceid=429, filesize=30.5 K 2024-12-05T12:14:19,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/470725ded1584a3fa3d0b79c67057d7b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/470725ded1584a3fa3d0b79c67057d7b 2024-12-05T12:14:19,233 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/470725ded1584a3fa3d0b79c67057d7b, entries=150, sequenceid=429, filesize=12.0 K 2024-12-05T12:14:19,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/ee47ebea4a584eed973d4d83a0105223 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ee47ebea4a584eed973d4d83a0105223 2024-12-05T12:14:19,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733400919253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:19,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:19,255 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ee47ebea4a584eed973d4d83a0105223, entries=150, sequenceid=429, filesize=12.0 K 2024-12-05T12:14:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48878 deadline: 1733400919253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:19,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:19,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:19,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:19,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:14:19,263 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dde686338ee027649f1546b042d1ca1b in 680ms, sequenceid=429, compaction requested=true 2024-12-05T12:14:19,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:19,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:19,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-05T12:14:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-05T12:14:19,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-05T12:14:19,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 838 msec 2024-12-05T12:14:19,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 844 msec 2024-12-05T12:14:19,315 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-12-05T12:14:19,316 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:19,316 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,316 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:19,316 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-12-05T12:14:19,316 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:19,316 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:19,317 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,327 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-12-05T12:14:19,327 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:19,327 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,327 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:19,328 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-12-05T12:14:19,328 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:19,328 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,328 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T12:14:19,564 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:19,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-05T12:14:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:14:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:14:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:14:19,566 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-12-05T12:14:19,566 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:19,566 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-12-05T12:14:19,566 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,566 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:19,566 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,566 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,566 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:19,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:19,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:19,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c1697abc138440159d08b6c79f6c74b0_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400858926/Put/seqid=0 2024-12-05T12:14:19,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742062_1238 (size=12454) 2024-12-05T12:14:19,577 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:19,582 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c1697abc138440159d08b6c79f6c74b0_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c1697abc138440159d08b6c79f6c74b0_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:19,583 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/cc647fb679504d8fb6869121ff781b15, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:19,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/cc647fb679504d8fb6869121ff781b15 is 175, key is test_row_0/A:col10/1733400858926/Put/seqid=0 2024-12-05T12:14:19,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742063_1239 (size=31255) 2024-12-05T12:14:19,990 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=467, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/cc647fb679504d8fb6869121ff781b15 2024-12-05T12:14:20,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/c7c18cd8c1794f29b5ba957eb0197ad9 is 50, key is test_row_0/B:col10/1733400858926/Put/seqid=0 2024-12-05T12:14:20,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742064_1240 (size=12301) 2024-12-05T12:14:20,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/c7c18cd8c1794f29b5ba957eb0197ad9 2024-12-05T12:14:20,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/bdbdb5e203854ee28a7e6ef4784b3b16 is 50, key is test_row_0/C:col10/1733400858926/Put/seqid=0 2024-12-05T12:14:20,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742065_1241 (size=12301) 2024-12-05T12:14:20,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/bdbdb5e203854ee28a7e6ef4784b3b16 2024-12-05T12:14:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/cc647fb679504d8fb6869121ff781b15 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/cc647fb679504d8fb6869121ff781b15 2024-12-05T12:14:20,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/cc647fb679504d8fb6869121ff781b15, entries=150, sequenceid=467, filesize=30.5 K 2024-12-05T12:14:20,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/c7c18cd8c1794f29b5ba957eb0197ad9 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/c7c18cd8c1794f29b5ba957eb0197ad9 2024-12-05T12:14:20,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/c7c18cd8c1794f29b5ba957eb0197ad9, entries=150, sequenceid=467, filesize=12.0 K 2024-12-05T12:14:20,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/bdbdb5e203854ee28a7e6ef4784b3b16 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bdbdb5e203854ee28a7e6ef4784b3b16 2024-12-05T12:14:20,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bdbdb5e203854ee28a7e6ef4784b3b16, entries=150, sequenceid=467, filesize=12.0 K 2024-12-05T12:14:20,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=0 B/0 for dde686338ee027649f1546b042d1ca1b in 897ms, sequenceid=467, compaction requested=true 2024-12-05T12:14:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:20,462 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:14:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:20,462 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:14:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde686338ee027649f1546b042d1ca1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:20,463 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:14:20,463 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125838 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:14:20,464 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/B is initiating minor compaction (all files) 2024-12-05T12:14:20,464 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/A is initiating minor compaction (all files) 2024-12-05T12:14:20,464 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/B in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:20,464 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/7511a51940ea41928b6e5cc661fde2a5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/6fea3d8297e14601999a7752833435f7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/470725ded1584a3fa3d0b79c67057d7b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/c7c18cd8c1794f29b5ba957eb0197ad9] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=48.8 K 2024-12-05T12:14:20,464 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/A in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:20,464 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/66151c266b0042019deda58d67dc9b10, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/f56697dc7bcb4053a0110eaae31a81cd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/716b506a450b42e1b23d509345948133, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/cc647fb679504d8fb6869121ff781b15] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=122.9 K 2024-12-05T12:14:20,464 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:20,464 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/66151c266b0042019deda58d67dc9b10, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/f56697dc7bcb4053a0110eaae31a81cd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/716b506a450b42e1b23d509345948133, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/cc647fb679504d8fb6869121ff781b15] 2024-12-05T12:14:20,464 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 7511a51940ea41928b6e5cc661fde2a5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733400855345 2024-12-05T12:14:20,465 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66151c266b0042019deda58d67dc9b10, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733400855345 2024-12-05T12:14:20,466 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56697dc7bcb4053a0110eaae31a81cd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733400856300 2024-12-05T12:14:20,466 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fea3d8297e14601999a7752833435f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733400856300 2024-12-05T12:14:20,467 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 470725ded1584a3fa3d0b79c67057d7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733400857555 2024-12-05T12:14:20,467 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 716b506a450b42e1b23d509345948133, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733400857555 2024-12-05T12:14:20,467 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting c7c18cd8c1794f29b5ba957eb0197ad9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733400858919 2024-12-05T12:14:20,468 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc647fb679504d8fb6869121ff781b15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733400858919 2024-12-05T12:14:20,501 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:20,506 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#B#compaction#198 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:20,507 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/3e224df8d920476984b874f54d8708d8 is 50, key is test_row_0/B:col10/1733400858926/Put/seqid=0 2024-12-05T12:14:20,516 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412056b57b10481bc47809ab92d4be5916bf5_dde686338ee027649f1546b042d1ca1b store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:20,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742066_1242 (size=13255) 2024-12-05T12:14:20,527 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/3e224df8d920476984b874f54d8708d8 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/3e224df8d920476984b874f54d8708d8 2024-12-05T12:14:20,534 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dde686338ee027649f1546b042d1ca1b/B of dde686338ee027649f1546b042d1ca1b into 3e224df8d920476984b874f54d8708d8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:20,534 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:20,534 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/B, priority=12, startTime=1733400860462; duration=0sec 2024-12-05T12:14:20,534 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:20,534 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:B 2024-12-05T12:14:20,534 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:14:20,539 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:14:20,539 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): dde686338ee027649f1546b042d1ca1b/C is initiating minor compaction (all files) 2024-12-05T12:14:20,539 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde686338ee027649f1546b042d1ca1b/C in TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:20,539 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8f49d1b7ff41465997c3bb8a398cf738, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/80cba62f513146009cc4d6bdf52daadc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ee47ebea4a584eed973d4d83a0105223, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bdbdb5e203854ee28a7e6ef4784b3b16] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp, totalSize=48.8 K 2024-12-05T12:14:20,540 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f49d1b7ff41465997c3bb8a398cf738, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733400855345 2024-12-05T12:14:20,542 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 80cba62f513146009cc4d6bdf52daadc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733400856300 2024-12-05T12:14:20,542 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412056b57b10481bc47809ab92d4be5916bf5_dde686338ee027649f1546b042d1ca1b, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:20,542 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056b57b10481bc47809ab92d4be5916bf5_dde686338ee027649f1546b042d1ca1b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:20,543 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting ee47ebea4a584eed973d4d83a0105223, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733400857555 2024-12-05T12:14:20,544 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting bdbdb5e203854ee28a7e6ef4784b3b16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733400858919 2024-12-05T12:14:20,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742067_1243 (size=4469) 2024-12-05T12:14:20,559 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#A#compaction#199 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:20,560 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/e573f89e6d6b4b2fa5adfd2df4ad54c4 is 175, key is test_row_0/A:col10/1733400858926/Put/seqid=0 2024-12-05T12:14:20,573 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde686338ee027649f1546b042d1ca1b#C#compaction#200 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:20,573 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/4adde22d74f245e0a96eaf577c26bcf9 is 50, key is test_row_0/C:col10/1733400858926/Put/seqid=0 2024-12-05T12:14:20,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742068_1244 (size=32209) 2024-12-05T12:14:20,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742069_1245 (size=13255) 2024-12-05T12:14:20,983 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/e573f89e6d6b4b2fa5adfd2df4ad54c4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e573f89e6d6b4b2fa5adfd2df4ad54c4 2024-12-05T12:14:20,988 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dde686338ee027649f1546b042d1ca1b/A of dde686338ee027649f1546b042d1ca1b into e573f89e6d6b4b2fa5adfd2df4ad54c4(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:20,988 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/4adde22d74f245e0a96eaf577c26bcf9 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/4adde22d74f245e0a96eaf577c26bcf9 2024-12-05T12:14:20,988 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:20,988 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/A, priority=12, startTime=1733400860462; duration=0sec 2024-12-05T12:14:20,988 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:20,988 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:A 2024-12-05T12:14:20,994 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dde686338ee027649f1546b042d1ca1b/C of dde686338ee027649f1546b042d1ca1b into 4adde22d74f245e0a96eaf577c26bcf9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:20,994 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde686338ee027649f1546b042d1ca1b: 2024-12-05T12:14:20,994 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., storeName=dde686338ee027649f1546b042d1ca1b/C, priority=12, startTime=1733400860462; duration=0sec 2024-12-05T12:14:20,994 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:20,994 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde686338ee027649f1546b042d1ca1b:C 2024-12-05T12:14:22,115 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-12-05T12:14:22,115 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:22,115 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:22,115 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:22,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:22,166 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-12-05T12:14:22,166 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:22,166 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:22,166 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:22,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:22,196 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-12-05T12:14:22,196 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:22,197 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:22,197 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:22,197 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:22,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T12:14:22,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-05T12:14:22,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 128 2024-12-05T12:14:22,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 174 2024-12-05T12:14:22,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-05T12:14:22,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-05T12:14:22,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T12:14:22,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2782 2024-12-05T12:14:22,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2721 2024-12-05T12:14:22,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T12:14:22,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1290 2024-12-05T12:14:22,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3863 rows 2024-12-05T12:14:22,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1270 2024-12-05T12:14:22,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3805 rows 2024-12-05T12:14:22,198 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:14:22,198 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:14:22,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:22,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:22,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:22,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T12:14:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T12:14:22,205 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400862205"}]},"ts":"1733400862205"} 2024-12-05T12:14:22,207 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T12:14:22,207 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T12:14:22,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T12:14:22,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, UNASSIGN}] 2024-12-05T12:14:22,210 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, UNASSIGN 2024-12-05T12:14:22,210 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=dde686338ee027649f1546b042d1ca1b, regionState=CLOSING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:22,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, UNASSIGN because future has completed 2024-12-05T12:14:22,212 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T12:14:22,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:14:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T12:14:22,364 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:22,364 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T12:14:22,364 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing dde686338ee027649f1546b042d1ca1b, disabling compactions & flushes 2024-12-05T12:14:22,364 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:22,364 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:22,364 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. after waiting 0 ms 2024-12-05T12:14:22,364 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:22,365 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing dde686338ee027649f1546b042d1ca1b 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-05T12:14:22,365 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=A 2024-12-05T12:14:22,365 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:22,365 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=B 2024-12-05T12:14:22,365 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:22,365 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dde686338ee027649f1546b042d1ca1b, store=C 2024-12-05T12:14:22,365 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:22,371 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205de872d93bd6945feae8364c35dc53ae3_dde686338ee027649f1546b042d1ca1b is 50, key is test_row_0/A:col10/1733400862194/Put/seqid=0 2024-12-05T12:14:22,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742070_1246 (size=9914) 2024-12-05T12:14:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T12:14:22,775 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:22,780 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205de872d93bd6945feae8364c35dc53ae3_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de872d93bd6945feae8364c35dc53ae3_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:22,781 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/9f1a65595cab4c2ca18db0a573f41f01, store: [table=TestAcidGuarantees family=A region=dde686338ee027649f1546b042d1ca1b] 2024-12-05T12:14:22,781 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/9f1a65595cab4c2ca18db0a573f41f01 is 175, key is test_row_0/A:col10/1733400862194/Put/seqid=0 2024-12-05T12:14:22,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742071_1247 (size=22561) 2024-12-05T12:14:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T12:14:23,189 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=476, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/9f1a65595cab4c2ca18db0a573f41f01 2024-12-05T12:14:23,197 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/bf52e2f344b4484f843d0402ec3c4d35 is 50, key is test_row_0/B:col10/1733400862194/Put/seqid=0 2024-12-05T12:14:23,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742072_1248 (size=9857) 2024-12-05T12:14:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T12:14:23,602 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/bf52e2f344b4484f843d0402ec3c4d35 2024-12-05T12:14:23,612 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/33d8ec7b68404601bb74c0a515e0ce87 is 50, key is test_row_0/C:col10/1733400862194/Put/seqid=0 2024-12-05T12:14:23,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742073_1249 (size=9857) 2024-12-05T12:14:24,017 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/33d8ec7b68404601bb74c0a515e0ce87 2024-12-05T12:14:24,023 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/A/9f1a65595cab4c2ca18db0a573f41f01 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9f1a65595cab4c2ca18db0a573f41f01 2024-12-05T12:14:24,027 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9f1a65595cab4c2ca18db0a573f41f01, entries=100, sequenceid=476, filesize=22.0 K 2024-12-05T12:14:24,028 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/B/bf52e2f344b4484f843d0402ec3c4d35 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/bf52e2f344b4484f843d0402ec3c4d35 2024-12-05T12:14:24,032 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/bf52e2f344b4484f843d0402ec3c4d35, entries=100, sequenceid=476, filesize=9.6 K 2024-12-05T12:14:24,033 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/.tmp/C/33d8ec7b68404601bb74c0a515e0ce87 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/33d8ec7b68404601bb74c0a515e0ce87 2024-12-05T12:14:24,037 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/33d8ec7b68404601bb74c0a515e0ce87, entries=100, sequenceid=476, filesize=9.6 K 2024-12-05T12:14:24,038 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for dde686338ee027649f1546b042d1ca1b in 1673ms, sequenceid=476, compaction requested=false 2024-12-05T12:14:24,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5e3bfdf189494a148e4028aeacfaef5a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3c815f3a2db342e5aa97ec54cfe3db92, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/acd48bf99ecc4f40bf8c36fed29a2ea3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/154544d526a6427da6e0acd90f3a3076, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e69d1515358248d68efe2c723a38989c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/0e795c8d66a24b65b24e313220274d60, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/000c9526dbfa477283788dfd1f7a85f2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c2e3dd708f8149c9a845535bc448071d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/39e6eaf74d114b91a64f858af878422e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/ecac83d0bba44e0c876b9486265f5998, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/70bef445702745979a2954bea8277de3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c6c5f7acbc3744b393dc69a233803171, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/004d5ec8388846cb83058fe3ef7e2bf6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9b58aae8c16d495bb28e6da0ec2cc90b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e3b80886ab3b4b279c66c9a7ae942703, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/83c814174e594c6b83f9ba3371171c81, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/fa04b0489c334afb9cc35a2afeadb6a2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/a5fbca39c080475f95608d4f9f80dfca, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/69db3a8806f64259a39282c126067a90, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/33ba843be340425d8ccf6b777863a083, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/029c942a9a7d4717882f19ee476cdb51, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/7a6b704720c442bba37371431e990194, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5742fb9780d74a2288a39a4beeeb7cb1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3ce5d3b9c4584be2bc6d5a4b36fdce64, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/66151c266b0042019deda58d67dc9b10, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/f56697dc7bcb4053a0110eaae31a81cd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/716b506a450b42e1b23d509345948133, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/cc647fb679504d8fb6869121ff781b15] to archive 2024-12-05T12:14:24,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:14:24,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5e3bfdf189494a148e4028aeacfaef5a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5e3bfdf189494a148e4028aeacfaef5a 2024-12-05T12:14:24,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3c815f3a2db342e5aa97ec54cfe3db92 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3c815f3a2db342e5aa97ec54cfe3db92 2024-12-05T12:14:24,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/acd48bf99ecc4f40bf8c36fed29a2ea3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/acd48bf99ecc4f40bf8c36fed29a2ea3 2024-12-05T12:14:24,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/154544d526a6427da6e0acd90f3a3076 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/154544d526a6427da6e0acd90f3a3076 2024-12-05T12:14:24,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e69d1515358248d68efe2c723a38989c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e69d1515358248d68efe2c723a38989c 2024-12-05T12:14:24,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/0e795c8d66a24b65b24e313220274d60 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/0e795c8d66a24b65b24e313220274d60 2024-12-05T12:14:24,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/000c9526dbfa477283788dfd1f7a85f2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/000c9526dbfa477283788dfd1f7a85f2 2024-12-05T12:14:24,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c2e3dd708f8149c9a845535bc448071d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c2e3dd708f8149c9a845535bc448071d 2024-12-05T12:14:24,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/39e6eaf74d114b91a64f858af878422e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/39e6eaf74d114b91a64f858af878422e 2024-12-05T12:14:24,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/ecac83d0bba44e0c876b9486265f5998 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/ecac83d0bba44e0c876b9486265f5998 2024-12-05T12:14:24,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/70bef445702745979a2954bea8277de3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/70bef445702745979a2954bea8277de3 2024-12-05T12:14:24,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c6c5f7acbc3744b393dc69a233803171 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/c6c5f7acbc3744b393dc69a233803171 2024-12-05T12:14:24,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/004d5ec8388846cb83058fe3ef7e2bf6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/004d5ec8388846cb83058fe3ef7e2bf6 2024-12-05T12:14:24,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9b58aae8c16d495bb28e6da0ec2cc90b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9b58aae8c16d495bb28e6da0ec2cc90b 2024-12-05T12:14:24,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e3b80886ab3b4b279c66c9a7ae942703 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e3b80886ab3b4b279c66c9a7ae942703 2024-12-05T12:14:24,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/83c814174e594c6b83f9ba3371171c81 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/83c814174e594c6b83f9ba3371171c81 2024-12-05T12:14:24,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/fa04b0489c334afb9cc35a2afeadb6a2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/fa04b0489c334afb9cc35a2afeadb6a2 2024-12-05T12:14:24,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/a5fbca39c080475f95608d4f9f80dfca to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/a5fbca39c080475f95608d4f9f80dfca 2024-12-05T12:14:24,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/69db3a8806f64259a39282c126067a90 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/69db3a8806f64259a39282c126067a90 2024-12-05T12:14:24,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/33ba843be340425d8ccf6b777863a083 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/33ba843be340425d8ccf6b777863a083 2024-12-05T12:14:24,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/029c942a9a7d4717882f19ee476cdb51 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/029c942a9a7d4717882f19ee476cdb51 2024-12-05T12:14:24,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/7a6b704720c442bba37371431e990194 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/7a6b704720c442bba37371431e990194 2024-12-05T12:14:24,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5742fb9780d74a2288a39a4beeeb7cb1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/5742fb9780d74a2288a39a4beeeb7cb1 2024-12-05T12:14:24,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3ce5d3b9c4584be2bc6d5a4b36fdce64 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/3ce5d3b9c4584be2bc6d5a4b36fdce64 2024-12-05T12:14:24,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/66151c266b0042019deda58d67dc9b10 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/66151c266b0042019deda58d67dc9b10 2024-12-05T12:14:24,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/f56697dc7bcb4053a0110eaae31a81cd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/f56697dc7bcb4053a0110eaae31a81cd 2024-12-05T12:14:24,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/716b506a450b42e1b23d509345948133 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/716b506a450b42e1b23d509345948133 2024-12-05T12:14:24,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/cc647fb679504d8fb6869121ff781b15 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/cc647fb679504d8fb6869121ff781b15 2024-12-05T12:14:24,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/935ba20078b148abafcb6c58962bca2d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/78965755cf0b4ab4a6f8ad352c91bb3c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/eb442a5ebefe499bbd86a50bd3f9fd3a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b7a451868de94989be9aba0c8d9bb974, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/41f5d64064164d53b22ad8a4697e8a03, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b116511ae15c4c319fa3c68a74555dce, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/f653b7fb38b648929402f783c6e78ff8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/84b7a8b0d0304e19a1ceb12af065820d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/864b3679c3524b8caa0924854bea028a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b09171de9ddc4efb8200debf0cb973d0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/cd4742d29d334f94b4c6c8ee508ab974, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/5a51017038124cad9e5faac5ab830b51, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29ef646a416d4e3cb87ab1e2c7f24759, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29aaec7831b746008c98b672c809999c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/5d99da7c00794ddaba4fbba645e994ea, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/9ba597ceccbf4d6ea02bbd6f6592b47e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/637acd565713477b9e88b3533c13eba1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/50a0ddd0d0e04da489bf3fae3eb5e6ee, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/a55303bf3d694db881f25ebfffdc65ef, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b70fe07e2d7e4d4bbf64c6c6d05ab167, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/672ed03175d04b478edcdbabf71e5739, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/ae371408532346abaae8c3809bbef411, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/00c525c81fa040ae9758995d36dbdb80, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/7511a51940ea41928b6e5cc661fde2a5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/38022f947f45408997d243dde6be4177, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/6fea3d8297e14601999a7752833435f7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/470725ded1584a3fa3d0b79c67057d7b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/c7c18cd8c1794f29b5ba957eb0197ad9] to archive 2024-12-05T12:14:24,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:14:24,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/935ba20078b148abafcb6c58962bca2d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/935ba20078b148abafcb6c58962bca2d 2024-12-05T12:14:24,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/78965755cf0b4ab4a6f8ad352c91bb3c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/78965755cf0b4ab4a6f8ad352c91bb3c 2024-12-05T12:14:24,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/eb442a5ebefe499bbd86a50bd3f9fd3a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/eb442a5ebefe499bbd86a50bd3f9fd3a 2024-12-05T12:14:24,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b7a451868de94989be9aba0c8d9bb974 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b7a451868de94989be9aba0c8d9bb974 2024-12-05T12:14:24,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/41f5d64064164d53b22ad8a4697e8a03 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/41f5d64064164d53b22ad8a4697e8a03 2024-12-05T12:14:24,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b116511ae15c4c319fa3c68a74555dce to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b116511ae15c4c319fa3c68a74555dce 2024-12-05T12:14:24,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/f653b7fb38b648929402f783c6e78ff8 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/f653b7fb38b648929402f783c6e78ff8 2024-12-05T12:14:24,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/84b7a8b0d0304e19a1ceb12af065820d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/84b7a8b0d0304e19a1ceb12af065820d 2024-12-05T12:14:24,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/864b3679c3524b8caa0924854bea028a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/864b3679c3524b8caa0924854bea028a 2024-12-05T12:14:24,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b09171de9ddc4efb8200debf0cb973d0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b09171de9ddc4efb8200debf0cb973d0 2024-12-05T12:14:24,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/cd4742d29d334f94b4c6c8ee508ab974 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/cd4742d29d334f94b4c6c8ee508ab974 2024-12-05T12:14:24,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/5a51017038124cad9e5faac5ab830b51 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/5a51017038124cad9e5faac5ab830b51 2024-12-05T12:14:24,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29ef646a416d4e3cb87ab1e2c7f24759 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29ef646a416d4e3cb87ab1e2c7f24759 2024-12-05T12:14:24,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29aaec7831b746008c98b672c809999c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/29aaec7831b746008c98b672c809999c 2024-12-05T12:14:24,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/5d99da7c00794ddaba4fbba645e994ea to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/5d99da7c00794ddaba4fbba645e994ea 2024-12-05T12:14:24,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/9ba597ceccbf4d6ea02bbd6f6592b47e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/9ba597ceccbf4d6ea02bbd6f6592b47e 2024-12-05T12:14:24,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/637acd565713477b9e88b3533c13eba1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/637acd565713477b9e88b3533c13eba1 2024-12-05T12:14:24,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/50a0ddd0d0e04da489bf3fae3eb5e6ee to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/50a0ddd0d0e04da489bf3fae3eb5e6ee 2024-12-05T12:14:24,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/a55303bf3d694db881f25ebfffdc65ef to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/a55303bf3d694db881f25ebfffdc65ef 2024-12-05T12:14:24,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b70fe07e2d7e4d4bbf64c6c6d05ab167 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/b70fe07e2d7e4d4bbf64c6c6d05ab167 2024-12-05T12:14:24,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/672ed03175d04b478edcdbabf71e5739 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/672ed03175d04b478edcdbabf71e5739 2024-12-05T12:14:24,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/ae371408532346abaae8c3809bbef411 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/ae371408532346abaae8c3809bbef411 2024-12-05T12:14:24,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/00c525c81fa040ae9758995d36dbdb80 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/00c525c81fa040ae9758995d36dbdb80 2024-12-05T12:14:24,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/7511a51940ea41928b6e5cc661fde2a5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/7511a51940ea41928b6e5cc661fde2a5 2024-12-05T12:14:24,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/38022f947f45408997d243dde6be4177 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/38022f947f45408997d243dde6be4177 2024-12-05T12:14:24,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/6fea3d8297e14601999a7752833435f7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/6fea3d8297e14601999a7752833435f7 2024-12-05T12:14:24,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/470725ded1584a3fa3d0b79c67057d7b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/470725ded1584a3fa3d0b79c67057d7b 2024-12-05T12:14:24,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/c7c18cd8c1794f29b5ba957eb0197ad9 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/c7c18cd8c1794f29b5ba957eb0197ad9 2024-12-05T12:14:24,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3aa587ee12d94dfb93118a95e7c8521a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a2833617a77344c3ad545497c6fb016c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/e7b8e6bc001f4a7aa2bc4d44e8a3b961, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d970355a78d40b2ba2f22fb166a16c6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/54165a72b9a748b9bba8454be91af37b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d34f34bc07e4b26b32e38ed9994bbab, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ea23cf21764c426aa8409da030e1fbb7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a17f06ad94494e869e70b5f2571ee6a1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3880494075004290aef17a068857dc1f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/19564e41352e4307bb568bf327388554, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/f70e46a1feeb4b20ad2917acdeeaa359, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a51bbb76e4814c8e93c8366b457c85e4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/25c28a47a9394592a6333b295cd5ce3c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/29db028e4ada46c38200f82df7c73cc8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bf5e245735c04f5bb3d3f3913eebfe96, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/5b1271a0d8564c61a31c55e5b8cd96b6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3295a9da8e9f4b518b64a5b8e3c1cbfc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/84b9c7adc8034e7baf4d656366621e86, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/350daa49690e4332abcb1af0682107ad, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/b849a62b3eb74673a4bbe2f94d25fa8f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/775c61c3cc804002bfe1805935fa9c7e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/d905228a16684386a272feb2bc86a0a0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/60e99cbb4fad455aa290d41857976183, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8f49d1b7ff41465997c3bb8a398cf738, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/df7f3a919e1c47aea60d357b20d54482, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/80cba62f513146009cc4d6bdf52daadc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ee47ebea4a584eed973d4d83a0105223, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bdbdb5e203854ee28a7e6ef4784b3b16] to archive 2024-12-05T12:14:24,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:14:24,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3aa587ee12d94dfb93118a95e7c8521a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3aa587ee12d94dfb93118a95e7c8521a 2024-12-05T12:14:24,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a2833617a77344c3ad545497c6fb016c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a2833617a77344c3ad545497c6fb016c 2024-12-05T12:14:24,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/e7b8e6bc001f4a7aa2bc4d44e8a3b961 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/e7b8e6bc001f4a7aa2bc4d44e8a3b961 2024-12-05T12:14:24,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d970355a78d40b2ba2f22fb166a16c6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d970355a78d40b2ba2f22fb166a16c6 2024-12-05T12:14:24,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/54165a72b9a748b9bba8454be91af37b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/54165a72b9a748b9bba8454be91af37b 2024-12-05T12:14:24,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d34f34bc07e4b26b32e38ed9994bbab to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8d34f34bc07e4b26b32e38ed9994bbab 2024-12-05T12:14:24,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ea23cf21764c426aa8409da030e1fbb7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ea23cf21764c426aa8409da030e1fbb7 2024-12-05T12:14:24,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a17f06ad94494e869e70b5f2571ee6a1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a17f06ad94494e869e70b5f2571ee6a1 2024-12-05T12:14:24,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3880494075004290aef17a068857dc1f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3880494075004290aef17a068857dc1f 2024-12-05T12:14:24,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/19564e41352e4307bb568bf327388554 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/19564e41352e4307bb568bf327388554 2024-12-05T12:14:24,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/f70e46a1feeb4b20ad2917acdeeaa359 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/f70e46a1feeb4b20ad2917acdeeaa359 2024-12-05T12:14:24,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a51bbb76e4814c8e93c8366b457c85e4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/a51bbb76e4814c8e93c8366b457c85e4 2024-12-05T12:14:24,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/25c28a47a9394592a6333b295cd5ce3c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/25c28a47a9394592a6333b295cd5ce3c 2024-12-05T12:14:24,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/29db028e4ada46c38200f82df7c73cc8 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/29db028e4ada46c38200f82df7c73cc8 2024-12-05T12:14:24,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bf5e245735c04f5bb3d3f3913eebfe96 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bf5e245735c04f5bb3d3f3913eebfe96 2024-12-05T12:14:24,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/5b1271a0d8564c61a31c55e5b8cd96b6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/5b1271a0d8564c61a31c55e5b8cd96b6 2024-12-05T12:14:24,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3295a9da8e9f4b518b64a5b8e3c1cbfc to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/3295a9da8e9f4b518b64a5b8e3c1cbfc 2024-12-05T12:14:24,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/84b9c7adc8034e7baf4d656366621e86 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/84b9c7adc8034e7baf4d656366621e86 2024-12-05T12:14:24,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/350daa49690e4332abcb1af0682107ad to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/350daa49690e4332abcb1af0682107ad 2024-12-05T12:14:24,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/b849a62b3eb74673a4bbe2f94d25fa8f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/b849a62b3eb74673a4bbe2f94d25fa8f 2024-12-05T12:14:24,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/775c61c3cc804002bfe1805935fa9c7e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/775c61c3cc804002bfe1805935fa9c7e 2024-12-05T12:14:24,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/d905228a16684386a272feb2bc86a0a0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/d905228a16684386a272feb2bc86a0a0 2024-12-05T12:14:24,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/60e99cbb4fad455aa290d41857976183 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/60e99cbb4fad455aa290d41857976183 2024-12-05T12:14:24,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8f49d1b7ff41465997c3bb8a398cf738 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/8f49d1b7ff41465997c3bb8a398cf738 2024-12-05T12:14:24,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/df7f3a919e1c47aea60d357b20d54482 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/df7f3a919e1c47aea60d357b20d54482 2024-12-05T12:14:24,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/80cba62f513146009cc4d6bdf52daadc to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/80cba62f513146009cc4d6bdf52daadc 2024-12-05T12:14:24,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ee47ebea4a584eed973d4d83a0105223 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/ee47ebea4a584eed973d4d83a0105223 2024-12-05T12:14:24,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bdbdb5e203854ee28a7e6ef4784b3b16 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/bdbdb5e203854ee28a7e6ef4784b3b16 2024-12-05T12:14:24,148 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/recovered.edits/479.seqid, newMaxSeqId=479, maxSeqId=4 2024-12-05T12:14:24,149 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. 2024-12-05T12:14:24,149 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for dde686338ee027649f1546b042d1ca1b: Waiting for close lock at 1733400862364Running coprocessor pre-close hooks at 1733400862364Disabling compacts and flushes for region at 1733400862364Disabling writes for close at 1733400862364Obtaining lock to block concurrent updates at 1733400862365 (+1 ms)Preparing flush snapshotting stores in dde686338ee027649f1546b042d1ca1b at 1733400862365Finished memstore snapshotting TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733400862365Flushing stores of TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b. at 1733400862366 (+1 ms)Flushing dde686338ee027649f1546b042d1ca1b/A: creating writer at 1733400862366Flushing mob file dde686338ee027649f1546b042d1ca1b/A: appending metadata at 1733400862371 (+5 ms)Flushing mob file dde686338ee027649f1546b042d1ca1b/A: closing flushed file at 1733400862371Flushing dde686338ee027649f1546b042d1ca1b/A: appending metadata at 1733400862781 (+410 ms)Flushing dde686338ee027649f1546b042d1ca1b/A: closing flushed file at 1733400862781Flushing dde686338ee027649f1546b042d1ca1b/B: creating writer at 1733400863193 (+412 ms)Flushing dde686338ee027649f1546b042d1ca1b/B: appending metadata at 1733400863197 (+4 ms)Flushing dde686338ee027649f1546b042d1ca1b/B: closing flushed file at 1733400863197Flushing dde686338ee027649f1546b042d1ca1b/C: creating writer at 1733400863608 (+411 ms)Flushing dde686338ee027649f1546b042d1ca1b/C: appending metadata at 1733400863611 (+3 ms)Flushing dde686338ee027649f1546b042d1ca1b/C: closing flushed file at 1733400863611Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@292307ce: reopening flushed file at 1733400864022 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1981bc3e: reopening flushed file at 1733400864027 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66307c30: reopening flushed file at 1733400864032 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for dde686338ee027649f1546b042d1ca1b in 1673ms, sequenceid=476, compaction requested=false at 1733400864038 (+6 ms)Writing region close event to WAL at 1733400864144 (+106 ms)Running coprocessor post-close hooks at 1733400864149 (+5 ms)Closed at 1733400864149 2024-12-05T12:14:24,151 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,151 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=dde686338ee027649f1546b042d1ca1b, regionState=CLOSED 2024-12-05T12:14:24,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:14:24,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-05T12:14:24,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure dde686338ee027649f1546b042d1ca1b, server=9ea9abad0e9b,44759,1733400796278 in 1.9410 sec 2024-12-05T12:14:24,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-05T12:14:24,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde686338ee027649f1546b042d1ca1b, UNASSIGN in 1.9460 sec 2024-12-05T12:14:24,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-05T12:14:24,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9490 sec 2024-12-05T12:14:24,159 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400864159"}]},"ts":"1733400864159"} 2024-12-05T12:14:24,160 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T12:14:24,161 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T12:14:24,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9620 sec 2024-12-05T12:14:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T12:14:24,353 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:24,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T12:14:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:24,356 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T12:14:24,357 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:24,360 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,364 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/recovered.edits] 2024-12-05T12:14:24,367 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9f1a65595cab4c2ca18db0a573f41f01 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/9f1a65595cab4c2ca18db0a573f41f01 2024-12-05T12:14:24,368 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e573f89e6d6b4b2fa5adfd2df4ad54c4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/A/e573f89e6d6b4b2fa5adfd2df4ad54c4 2024-12-05T12:14:24,372 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/3e224df8d920476984b874f54d8708d8 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/3e224df8d920476984b874f54d8708d8 2024-12-05T12:14:24,376 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/bf52e2f344b4484f843d0402ec3c4d35 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/B/bf52e2f344b4484f843d0402ec3c4d35 2024-12-05T12:14:24,381 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/33d8ec7b68404601bb74c0a515e0ce87 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/33d8ec7b68404601bb74c0a515e0ce87 2024-12-05T12:14:24,382 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/4adde22d74f245e0a96eaf577c26bcf9 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/C/4adde22d74f245e0a96eaf577c26bcf9 2024-12-05T12:14:24,386 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/recovered.edits/479.seqid to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b/recovered.edits/479.seqid 2024-12-05T12:14:24,387 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,387 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T12:14:24,387 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T12:14:24,388 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T12:14:24,393 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050575d94d481a414a84791d12ccd32243_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050575d94d481a414a84791d12ccd32243_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,397 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205058b373c445748a79f07318f7f2cde51_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205058b373c445748a79f07318f7f2cde51_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,398 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050a5ec60d3c794ecf9a87399d531faf00_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050a5ec60d3c794ecf9a87399d531faf00_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,400 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053934695d9c934fd5a4875c99229a3cb6_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053934695d9c934fd5a4875c99229a3cb6_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,402 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120547a3d211402e4c3b91dc51cbe5df0fec_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120547a3d211402e4c3b91dc51cbe5df0fec_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,405 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205641dc62b1b8b4fa9b15819bfffc3cf39_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205641dc62b1b8b4fa9b15819bfffc3cf39_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,424 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120568072543768e4321abcf6a12cdf218ba_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120568072543768e4321abcf6a12cdf218ba_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,426 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057d32506d266b44e1995b2c1c5f84d26c_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057d32506d266b44e1995b2c1c5f84d26c_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,428 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057d60d0a95fe644bbb3a5d808f6225a98_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057d60d0a95fe644bbb3a5d808f6225a98_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,431 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058f6ae3cca4914cb29c46b5f15939dacd_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058f6ae3cca4914cb29c46b5f15939dacd_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,434 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205973517785e394cfaad005a2c6b85cdc8_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205973517785e394cfaad005a2c6b85cdc8_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,437 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b543fbb975ac468b847804eb174144d8_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b543fbb975ac468b847804eb174144d8_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,439 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b79deaa4d68749f788e8ea745b96a3cc_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b79deaa4d68749f788e8ea745b96a3cc_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,450 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b7c5943ba1e84ac4a57a7ae078bdb5b2_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b7c5943ba1e84ac4a57a7ae078bdb5b2_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,452 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c1697abc138440159d08b6c79f6c74b0_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c1697abc138440159d08b6c79f6c74b0_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,453 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c6ce934f267c48c2b768baa4f2ff7e78_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c6ce934f267c48c2b768baa4f2ff7e78_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,455 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d5456588ddb141ad9cce64c99e3e161f_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d5456588ddb141ad9cce64c99e3e161f_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,457 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d981c1c707014130b09c6e19600e26e5_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d981c1c707014130b09c6e19600e26e5_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,458 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205dbe40f9f3c204ed1825f14854b1280d0_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205dbe40f9f3c204ed1825f14854b1280d0_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,460 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de872d93bd6945feae8364c35dc53ae3_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de872d93bd6945feae8364c35dc53ae3_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,462 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f65a281a98514f98abc1a011001b31e3_dde686338ee027649f1546b042d1ca1b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f65a281a98514f98abc1a011001b31e3_dde686338ee027649f1546b042d1ca1b 2024-12-05T12:14:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T12:14:24,464 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T12:14:24,467 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:24,471 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T12:14:24,474 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T12:14:24,476 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:24,476 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T12:14:24,476 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733400864476"}]},"ts":"9223372036854775807"} 2024-12-05T12:14:24,479 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T12:14:24,479 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dde686338ee027649f1546b042d1ca1b, NAME => 'TestAcidGuarantees,,1733400836084.dde686338ee027649f1546b042d1ca1b.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T12:14:24,479 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T12:14:24,479 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733400864479"}]},"ts":"9223372036854775807"} 2024-12-05T12:14:24,482 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T12:14:24,482 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:24,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 128 msec 2024-12-05T12:14:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T12:14:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T12:14:24,674 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:24,686 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 248) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/cluster_9f95a8c6-304c-dd62-ec56-918ba890d196/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-660316091_22 at /127.0.0.1:55282 [Waiting for operation #170] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-660316091_22 at /127.0.0.1:42564 [Waiting for operation #456] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/cluster_9f95a8c6-304c-dd62-ec56-918ba890d196/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-144804023_22 at /127.0.0.1:55274 [Waiting for operation #279] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-144804023_22 at /127.0.0.1:42652 [Waiting for operation #442] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=705 (was 576) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4475 (was 3641) - AvailableMemoryMB LEAK? - 2024-12-05T12:14:24,694 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=705, ProcessCount=11, AvailableMemoryMB=4475 2024-12-05T12:14:24,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T12:14:24,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:14:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:24,697 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T12:14:24,698 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:24,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-12-05T12:14:24,698 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T12:14:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T12:14:24,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742074_1250 (size=1061) 2024-12-05T12:14:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T12:14:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T12:14:25,106 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e 2024-12-05T12:14:25,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742075_1251 (size=53) 2024-12-05T12:14:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T12:14:25,513 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:14:25,513 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1509b956c8790d96ce37e7755c7bf6f2, disabling compactions & flushes 2024-12-05T12:14:25,513 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:25,513 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:25,513 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. after waiting 0 ms 2024-12-05T12:14:25,513 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:25,513 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:25,513 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1509b956c8790d96ce37e7755c7bf6f2: Waiting for close lock at 1733400865513Disabling compacts and flushes for region at 1733400865513Disabling writes for close at 1733400865513Writing region close event to WAL at 1733400865513Closed at 1733400865513 2024-12-05T12:14:25,515 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T12:14:25,515 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733400865515"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733400865515"}]},"ts":"1733400865515"} 2024-12-05T12:14:25,516 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T12:14:25,517 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T12:14:25,517 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400865517"}]},"ts":"1733400865517"} 2024-12-05T12:14:25,519 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T12:14:25,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1509b956c8790d96ce37e7755c7bf6f2, ASSIGN}] 2024-12-05T12:14:25,520 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1509b956c8790d96ce37e7755c7bf6f2, ASSIGN 2024-12-05T12:14:25,521 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1509b956c8790d96ce37e7755c7bf6f2, ASSIGN; state=OFFLINE, location=9ea9abad0e9b,44759,1733400796278; forceNewPlan=false, retain=false 2024-12-05T12:14:25,672 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1509b956c8790d96ce37e7755c7bf6f2, regionState=OPENING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:25,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1509b956c8790d96ce37e7755c7bf6f2, ASSIGN because future has completed 2024-12-05T12:14:25,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:14:25,832 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:25,833 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:14:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T12:14:25,833 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,833 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:14:25,833 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,834 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,835 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,836 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:14:25,837 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1509b956c8790d96ce37e7755c7bf6f2 columnFamilyName A 2024-12-05T12:14:25,837 DEBUG [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:25,837 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] regionserver.HStore(327): Store=1509b956c8790d96ce37e7755c7bf6f2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:14:25,837 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,838 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:14:25,838 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1509b956c8790d96ce37e7755c7bf6f2 columnFamilyName B 2024-12-05T12:14:25,838 DEBUG [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:25,838 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] regionserver.HStore(327): Store=1509b956c8790d96ce37e7755c7bf6f2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:14:25,838 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,839 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:14:25,839 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1509b956c8790d96ce37e7755c7bf6f2 columnFamilyName C 2024-12-05T12:14:25,839 DEBUG [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:25,840 INFO [StoreOpener-1509b956c8790d96ce37e7755c7bf6f2-1 {}] regionserver.HStore(327): Store=1509b956c8790d96ce37e7755c7bf6f2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:14:25,840 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:25,840 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,841 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,841 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,842 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,842 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,842 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:14:25,843 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,845 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:14:25,845 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 1509b956c8790d96ce37e7755c7bf6f2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68155871, jitterRate=0.015601620078086853}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:14:25,845 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:25,846 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 1509b956c8790d96ce37e7755c7bf6f2: Running coprocessor pre-open hook at 1733400865834Writing region info on filesystem at 1733400865834Initializing all the Stores at 1733400865835 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400865835Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400865835Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400865835Cleaning up temporary data from old regions at 1733400865842 (+7 ms)Running coprocessor post-open hooks at 1733400865846 (+4 ms)Region opened successfully at 1733400865846 2024-12-05T12:14:25,847 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., pid=65, masterSystemTime=1733400865827 2024-12-05T12:14:25,849 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:25,849 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:25,849 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1509b956c8790d96ce37e7755c7bf6f2, regionState=OPEN, openSeqNum=2, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:25,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:14:25,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-05T12:14:25,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 in 177 msec 2024-12-05T12:14:25,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-05T12:14:25,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1509b956c8790d96ce37e7755c7bf6f2, ASSIGN in 334 msec 2024-12-05T12:14:25,856 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T12:14:25,856 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400865856"}]},"ts":"1733400865856"} 2024-12-05T12:14:25,857 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T12:14:25,859 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T12:14:25,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-12-05T12:14:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T12:14:26,843 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:26,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bdc5a1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,845 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36068191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,847 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f8f317, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,848 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,852 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T12:14:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ea9abad0e9b,34103,1733400795471 2024-12-05T12:14:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@529bb36a 2024-12-05T12:14:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T12:14:26,854 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T12:14:26,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20832dd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,856 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99daac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1576d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb66027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,861 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4edec135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,862 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b55d68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b4a085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,865 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1164728c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,867 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227f4bd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f8f105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,869 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4edd06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,871 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@390d59fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aaa706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,873 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194fae2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,874 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3687dfd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417b74fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,876 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44053dcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,877 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12456d5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0a17bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,878 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209056d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,880 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@613882ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb595da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,881 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae3e7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,883 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d78ef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@262694dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,885 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a1e5fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,886 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ace55a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df95348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:26,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:26,888 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f77168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:26,889 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60eb9bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:26,890 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,890 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,890 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:26,890 DEBUG [Thread-1175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,891 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,891 DEBUG [Thread-1181 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,891 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,891 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-05T12:14:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T12:14:26,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:14:26,894 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:14:26,897 DEBUG [Thread-1183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:26,897 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:14:26,898 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,898 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:26,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:14:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:14:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:14:26,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:14:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:14:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:26,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:14:26,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:26,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:26,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:26,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:26,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:26,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400926932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:26,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/3a870d26aa3441e4b5fccf8297113f4f is 50, key is test_row_0/A:col10/1733400866906/Put/seqid=0 2024-12-05T12:14:26,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400926935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:26,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:26,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:26,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:26,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400926935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:26,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400926937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:26,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400926944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:26,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742076_1252 (size=12001) 2024-12-05T12:14:26,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/3a870d26aa3441e4b5fccf8297113f4f 2024-12-05T12:14:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T12:14:27,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/dc7843a9ea8d4cf6be581191fb8da9b8 is 50, key is test_row_0/B:col10/1733400866906/Put/seqid=0 2024-12-05T12:14:27,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742077_1253 (size=12001) 2024-12-05T12:14:27,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/dc7843a9ea8d4cf6be581191fb8da9b8 2024-12-05T12:14:27,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400927043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400927054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400927054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400927054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400927055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/755cdecae8734a05876e2c837a395cef is 50, key is test_row_0/C:col10/1733400866906/Put/seqid=0 2024-12-05T12:14:27,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742078_1254 (size=12001) 2024-12-05T12:14:27,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T12:14:27,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400927252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400927263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400927263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400927263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400927263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/755cdecae8734a05876e2c837a395cef 2024-12-05T12:14:27,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/3a870d26aa3441e4b5fccf8297113f4f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/3a870d26aa3441e4b5fccf8297113f4f 2024-12-05T12:14:27,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/3a870d26aa3441e4b5fccf8297113f4f, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T12:14:27,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/dc7843a9ea8d4cf6be581191fb8da9b8 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/dc7843a9ea8d4cf6be581191fb8da9b8 2024-12-05T12:14:27,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:27,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:27,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T12:14:27,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/dc7843a9ea8d4cf6be581191fb8da9b8, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T12:14:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/755cdecae8734a05876e2c837a395cef as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/755cdecae8734a05876e2c837a395cef 2024-12-05T12:14:27,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/755cdecae8734a05876e2c837a395cef, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T12:14:27,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1509b956c8790d96ce37e7755c7bf6f2 in 637ms, sequenceid=13, compaction requested=false 2024-12-05T12:14:27,543 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T12:14:27,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:27,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400866927, max timestamp=1733400867566Num uniques 859; 2024-12-05T12:14:27,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:27,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400866927, max timestamp=1733400867566Num uniques 859; 2024-12-05T12:14:27,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:27,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400866927, max timestamp=1733400867566Num uniques 859; 2024-12-05T12:14:27,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:27,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-05T12:14:27,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:27,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:27,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:27,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:27,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:27,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:27,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/b95aa8a204434562a6316ad0a96ba520 is 50, key is test_row_0/A:col10/1733400867579/Put/seqid=0 2024-12-05T12:14:27,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742079_1255 (size=16681) 2024-12-05T12:14:27,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/b95aa8a204434562a6316ad0a96ba520 2024-12-05T12:14:27,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400927611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400927611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400927613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400927620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400927621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/cb5195087fd747a5a5769dfc6be59215 is 50, key is test_row_0/B:col10/1733400867579/Put/seqid=0 2024-12-05T12:14:27,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:27,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:27,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400927723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742080_1256 (size=12001) 2024-12-05T12:14:27,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/cb5195087fd747a5a5769dfc6be59215 2024-12-05T12:14:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400927733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400927733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400927733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400927733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/3de7947ec8904206ae58b052236755c6 is 50, key is test_row_0/C:col10/1733400867579/Put/seqid=0 2024-12-05T12:14:27,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742081_1257 (size=12001) 2024-12-05T12:14:27,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:27,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:27,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400927933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400927943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400927943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400927943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400927944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:27,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:27,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:27,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:27,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T12:14:28,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:28,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:28,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/3de7947ec8904206ae58b052236755c6 2024-12-05T12:14:28,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/b95aa8a204434562a6316ad0a96ba520 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b95aa8a204434562a6316ad0a96ba520 2024-12-05T12:14:28,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b95aa8a204434562a6316ad0a96ba520, entries=250, sequenceid=40, filesize=16.3 K 2024-12-05T12:14:28,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/cb5195087fd747a5a5769dfc6be59215 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cb5195087fd747a5a5769dfc6be59215 2024-12-05T12:14:28,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cb5195087fd747a5a5769dfc6be59215, entries=150, sequenceid=40, filesize=11.7 K 2024-12-05T12:14:28,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/3de7947ec8904206ae58b052236755c6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/3de7947ec8904206ae58b052236755c6 2024-12-05T12:14:28,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/3de7947ec8904206ae58b052236755c6, entries=150, sequenceid=40, filesize=11.7 K 2024-12-05T12:14:28,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.72 KB/190176, currentSize=114.05 KB/116790 for 1509b956c8790d96ce37e7755c7bf6f2 in 613ms, sequenceid=40, compaction requested=false 2024-12-05T12:14:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:28,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T12:14:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:28,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/be5ffbfdffb44545b4226ea729a00d23 is 50, key is test_row_0/A:col10/1733400867617/Put/seqid=0 2024-12-05T12:14:28,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742082_1258 (size=12001) 2024-12-05T12:14:28,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400928260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400928261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400928264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400928265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400928266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:28,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:28,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400928373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400928373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400928373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400928373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400928373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:28,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:28,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400928583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400928583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400928583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400928583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400928583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:28,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:28,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/be5ffbfdffb44545b4226ea729a00d23 2024-12-05T12:14:28,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/6ebd5d47939245a58eafa25f37e41bd4 is 50, key is test_row_0/B:col10/1733400867617/Put/seqid=0 2024-12-05T12:14:28,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742083_1259 (size=12001) 2024-12-05T12:14:28,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/6ebd5d47939245a58eafa25f37e41bd4 2024-12-05T12:14:28,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/f05fc69b7307451ca5c5f963dc465824 is 50, key is test_row_0/C:col10/1733400867617/Put/seqid=0 2024-12-05T12:14:28,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742084_1260 (size=12001) 2024-12-05T12:14:28,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/f05fc69b7307451ca5c5f963dc465824 2024-12-05T12:14:28,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:28,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:28,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/be5ffbfdffb44545b4226ea729a00d23 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/be5ffbfdffb44545b4226ea729a00d23 2024-12-05T12:14:28,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/be5ffbfdffb44545b4226ea729a00d23, entries=150, sequenceid=61, filesize=11.7 K 2024-12-05T12:14:28,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/6ebd5d47939245a58eafa25f37e41bd4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/6ebd5d47939245a58eafa25f37e41bd4 2024-12-05T12:14:28,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/6ebd5d47939245a58eafa25f37e41bd4, entries=150, sequenceid=61, filesize=11.7 K 2024-12-05T12:14:28,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/f05fc69b7307451ca5c5f963dc465824 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/f05fc69b7307451ca5c5f963dc465824 2024-12-05T12:14:28,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/f05fc69b7307451ca5c5f963dc465824, entries=150, sequenceid=61, filesize=11.7 K 2024-12-05T12:14:28,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 1509b956c8790d96ce37e7755c7bf6f2 in 538ms, sequenceid=61, compaction requested=true 2024-12-05T12:14:28,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:28,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:28,783 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:28,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:28,783 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:28,787 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:28,787 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/B is initiating minor compaction (all files) 2024-12-05T12:14:28,787 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/B in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,787 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/dc7843a9ea8d4cf6be581191fb8da9b8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cb5195087fd747a5a5769dfc6be59215, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/6ebd5d47939245a58eafa25f37e41bd4] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=35.2 K 2024-12-05T12:14:28,787 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:28,788 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/A is initiating minor compaction (all files) 2024-12-05T12:14:28,788 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/A in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,788 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/3a870d26aa3441e4b5fccf8297113f4f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b95aa8a204434562a6316ad0a96ba520, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/be5ffbfdffb44545b4226ea729a00d23] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=39.7 K 2024-12-05T12:14:28,788 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting dc7843a9ea8d4cf6be581191fb8da9b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733400866901 2024-12-05T12:14:28,788 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a870d26aa3441e4b5fccf8297113f4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733400866901 2024-12-05T12:14:28,789 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting cb5195087fd747a5a5769dfc6be59215, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733400867577 2024-12-05T12:14:28,789 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b95aa8a204434562a6316ad0a96ba520, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733400866927 2024-12-05T12:14:28,789 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ebd5d47939245a58eafa25f37e41bd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400867609 2024-12-05T12:14:28,790 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting be5ffbfdffb44545b4226ea729a00d23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400867609 2024-12-05T12:14:28,836 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#A#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:28,837 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/503e31a5dac84bbd8ddc76a070666c26 is 50, key is test_row_0/A:col10/1733400867617/Put/seqid=0 2024-12-05T12:14:28,850 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:28,850 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/afe2245c2ac948c3a2f70c1a3c925a2b is 50, key is test_row_0/B:col10/1733400867617/Put/seqid=0 2024-12-05T12:14:28,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742085_1261 (size=12104) 2024-12-05T12:14:28,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742086_1262 (size=12104) 2024-12-05T12:14:28,884 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/afe2245c2ac948c3a2f70c1a3c925a2b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/afe2245c2ac948c3a2f70c1a3c925a2b 2024-12-05T12:14:28,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T12:14:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:28,900 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/B of 1509b956c8790d96ce37e7755c7bf6f2 into afe2245c2ac948c3a2f70c1a3c925a2b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:28,900 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:28,900 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/B, priority=13, startTime=1733400868783; duration=0sec 2024-12-05T12:14:28,900 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:28,900 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:B 2024-12-05T12:14:28,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:28,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:28,900 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:28,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:28,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:28,902 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:28,902 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/C is initiating minor compaction (all files) 2024-12-05T12:14:28,902 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/C in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,902 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/755cdecae8734a05876e2c837a395cef, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/3de7947ec8904206ae58b052236755c6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/f05fc69b7307451ca5c5f963dc465824] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=35.2 K 2024-12-05T12:14:28,902 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 755cdecae8734a05876e2c837a395cef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733400866901 2024-12-05T12:14:28,903 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de7947ec8904206ae58b052236755c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733400867577 2024-12-05T12:14:28,903 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting f05fc69b7307451ca5c5f963dc465824, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400867609 2024-12-05T12:14:28,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:28,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:28,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:28,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:28,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/28252c3f9cb54cf99c135e9cf6bf8e8f is 50, key is test_row_0/A:col10/1733400868895/Put/seqid=0 2024-12-05T12:14:28,918 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:28,919 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/d14fdfab8fd74efb917fe458d54c3700 is 50, key is test_row_0/C:col10/1733400867617/Put/seqid=0 2024-12-05T12:14:28,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400928922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400928922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400928921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400928924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400928925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:28,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742087_1263 (size=12001) 2024-12-05T12:14:28,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742088_1264 (size=12104) 2024-12-05T12:14:28,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/28252c3f9cb54cf99c135e9cf6bf8e8f 2024-12-05T12:14:28,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/77b6abcc2a144140b593b58b47247a5e is 50, key is test_row_0/B:col10/1733400868895/Put/seqid=0 2024-12-05T12:14:28,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742089_1265 (size=12001) 2024-12-05T12:14:28,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/77b6abcc2a144140b593b58b47247a5e 2024-12-05T12:14:28,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/a350d7f6dab8466491b726253c43c756 is 50, key is test_row_0/C:col10/1733400868895/Put/seqid=0 2024-12-05T12:14:29,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742090_1266 (size=12001) 2024-12-05T12:14:29,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400929032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400929033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400929033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400929033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400929033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T12:14:29,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:29,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:29,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:29,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:29,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:29,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:29,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400929243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400929243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400929243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400929243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400929243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,261 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/503e31a5dac84bbd8ddc76a070666c26 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/503e31a5dac84bbd8ddc76a070666c26 2024-12-05T12:14:29,266 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/A of 1509b956c8790d96ce37e7755c7bf6f2 into 503e31a5dac84bbd8ddc76a070666c26(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:29,266 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:29,266 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/A, priority=13, startTime=1733400868783; duration=0sec 2024-12-05T12:14:29,266 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:29,266 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:A 2024-12-05T12:14:29,351 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/d14fdfab8fd74efb917fe458d54c3700 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/d14fdfab8fd74efb917fe458d54c3700 2024-12-05T12:14:29,355 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/C of 1509b956c8790d96ce37e7755c7bf6f2 into d14fdfab8fd74efb917fe458d54c3700(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:29,355 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:29,355 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/C, priority=13, startTime=1733400868784; duration=0sec 2024-12-05T12:14:29,355 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:29,355 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:C 2024-12-05T12:14:29,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:29,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:29,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:29,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:29,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:29,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:29,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/a350d7f6dab8466491b726253c43c756 2024-12-05T12:14:29,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/28252c3f9cb54cf99c135e9cf6bf8e8f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/28252c3f9cb54cf99c135e9cf6bf8e8f 2024-12-05T12:14:29,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/28252c3f9cb54cf99c135e9cf6bf8e8f, entries=150, sequenceid=77, filesize=11.7 K 2024-12-05T12:14:29,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/77b6abcc2a144140b593b58b47247a5e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77b6abcc2a144140b593b58b47247a5e 2024-12-05T12:14:29,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77b6abcc2a144140b593b58b47247a5e, entries=150, sequenceid=77, filesize=11.7 K 2024-12-05T12:14:29,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/a350d7f6dab8466491b726253c43c756 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a350d7f6dab8466491b726253c43c756 2024-12-05T12:14:29,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a350d7f6dab8466491b726253c43c756, entries=150, sequenceid=77, filesize=11.7 K 2024-12-05T12:14:29,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1509b956c8790d96ce37e7755c7bf6f2 in 544ms, sequenceid=77, compaction requested=false 2024-12-05T12:14:29,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:29,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T12:14:29,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:29,535 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T12:14:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:29,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/b92ae98d4e5d44fd922fe3af469f66a5 is 50, key is test_row_0/A:col10/1733400868917/Put/seqid=0 2024-12-05T12:14:29,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:29,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742091_1267 (size=12001) 2024-12-05T12:14:29,557 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/b92ae98d4e5d44fd922fe3af469f66a5 2024-12-05T12:14:29,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/a6efd3ef77744668ad029a35a43081f9 is 50, key is test_row_0/B:col10/1733400868917/Put/seqid=0 2024-12-05T12:14:29,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400929567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400929567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400929569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400929570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400929575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742092_1268 (size=12001) 2024-12-05T12:14:29,674 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T12:14:29,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400929690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400929690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400929690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400929690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400929692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400929893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400929903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400929906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400929907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400929907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:29,987 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/a6efd3ef77744668ad029a35a43081f9 2024-12-05T12:14:30,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/1d7d93d9aa3a4705826ad34ab744bc39 is 50, key is test_row_0/C:col10/1733400868917/Put/seqid=0 2024-12-05T12:14:30,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742093_1269 (size=12001) 2024-12-05T12:14:30,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400930203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400930222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400930228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400930228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400930228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,449 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/1d7d93d9aa3a4705826ad34ab744bc39 2024-12-05T12:14:30,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/b92ae98d4e5d44fd922fe3af469f66a5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b92ae98d4e5d44fd922fe3af469f66a5 2024-12-05T12:14:30,461 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b92ae98d4e5d44fd922fe3af469f66a5, entries=150, sequenceid=96, filesize=11.7 K 2024-12-05T12:14:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/a6efd3ef77744668ad029a35a43081f9 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/a6efd3ef77744668ad029a35a43081f9 2024-12-05T12:14:30,470 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/a6efd3ef77744668ad029a35a43081f9, entries=150, sequenceid=96, filesize=11.7 K 2024-12-05T12:14:30,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/1d7d93d9aa3a4705826ad34ab744bc39 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1d7d93d9aa3a4705826ad34ab744bc39 2024-12-05T12:14:30,480 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1d7d93d9aa3a4705826ad34ab744bc39, entries=150, sequenceid=96, filesize=11.7 K 2024-12-05T12:14:30,481 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1509b956c8790d96ce37e7755c7bf6f2 in 946ms, sequenceid=96, compaction requested=true 2024-12-05T12:14:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-05T12:14:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-05T12:14:30,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-05T12:14:30,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5890 sec 2024-12-05T12:14:30,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 3.6000 sec 2024-12-05T12:14:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:30,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T12:14:30,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:30,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:30,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:30,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:30,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:30,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:30,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/a0a52150e76f4a3db06cf984a7d886bc is 50, key is test_row_0/A:col10/1733400869564/Put/seqid=0 2024-12-05T12:14:30,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400930754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742094_1270 (size=14341) 2024-12-05T12:14:30,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400930754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400930755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400930755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/a0a52150e76f4a3db06cf984a7d886bc 2024-12-05T12:14:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400930761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/b82b48927d0f41dab174718ae8b610ae is 50, key is test_row_0/B:col10/1733400869564/Put/seqid=0 2024-12-05T12:14:30,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742095_1271 (size=12001) 2024-12-05T12:14:30,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400930867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400930871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400930870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400930873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:30,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400930873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400931074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T12:14:31,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400931083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400931083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400931083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,091 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:31,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400931093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-05T12:14:31,100 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:31,101 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:31,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T12:14:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T12:14:31,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/b82b48927d0f41dab174718ae8b610ae 2024-12-05T12:14:31,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T12:14:31,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:31,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/741fa51380fc4a1898aea9ac930cb704 is 50, key is test_row_0/C:col10/1733400869564/Put/seqid=0 2024-12-05T12:14:31,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742096_1272 (size=12001) 2024-12-05T12:14:31,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400931397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400931397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400931397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400931399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400931403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T12:14:31,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:31,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T12:14:31,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T12:14:31,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/741fa51380fc4a1898aea9ac930cb704 2024-12-05T12:14:31,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/a0a52150e76f4a3db06cf984a7d886bc as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a0a52150e76f4a3db06cf984a7d886bc 2024-12-05T12:14:31,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a0a52150e76f4a3db06cf984a7d886bc, entries=200, sequenceid=112, filesize=14.0 K 2024-12-05T12:14:31,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/b82b48927d0f41dab174718ae8b610ae as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/b82b48927d0f41dab174718ae8b610ae 2024-12-05T12:14:31,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/b82b48927d0f41dab174718ae8b610ae, entries=150, sequenceid=112, filesize=11.7 K 2024-12-05T12:14:31,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/741fa51380fc4a1898aea9ac930cb704 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/741fa51380fc4a1898aea9ac930cb704 2024-12-05T12:14:31,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T12:14:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:31,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/741fa51380fc4a1898aea9ac930cb704, entries=150, sequenceid=112, filesize=11.7 K 2024-12-05T12:14:31,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1509b956c8790d96ce37e7755c7bf6f2 in 1006ms, sequenceid=112, compaction requested=true 2024-12-05T12:14:31,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:31,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:31,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:31,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:31,739 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:14:31,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:31,739 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:14:31,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:31,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:31,740 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:14:31,740 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/A is initiating minor compaction (all files) 2024-12-05T12:14:31,740 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/A in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,740 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/503e31a5dac84bbd8ddc76a070666c26, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/28252c3f9cb54cf99c135e9cf6bf8e8f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b92ae98d4e5d44fd922fe3af469f66a5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a0a52150e76f4a3db06cf984a7d886bc] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=49.3 K 2024-12-05T12:14:31,740 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:14:31,740 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/B is initiating minor compaction (all files) 2024-12-05T12:14:31,741 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 503e31a5dac84bbd8ddc76a070666c26, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400867609 2024-12-05T12:14:31,741 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/B in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,741 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/afe2245c2ac948c3a2f70c1a3c925a2b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77b6abcc2a144140b593b58b47247a5e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/a6efd3ef77744668ad029a35a43081f9, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/b82b48927d0f41dab174718ae8b610ae] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=47.0 K 2024-12-05T12:14:31,741 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28252c3f9cb54cf99c135e9cf6bf8e8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733400868895 2024-12-05T12:14:31,741 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b92ae98d4e5d44fd922fe3af469f66a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733400868917 2024-12-05T12:14:31,741 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting afe2245c2ac948c3a2f70c1a3c925a2b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400867609 2024-12-05T12:14:31,741 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0a52150e76f4a3db06cf984a7d886bc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400869564 2024-12-05T12:14:31,741 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 77b6abcc2a144140b593b58b47247a5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733400868895 2024-12-05T12:14:31,742 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting a6efd3ef77744668ad029a35a43081f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733400868917 2024-12-05T12:14:31,742 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b82b48927d0f41dab174718ae8b610ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400869564 2024-12-05T12:14:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T12:14:31,750 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:31,751 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/130aa228e2a94a858f69313b77c73cd7 is 50, key is test_row_0/A:col10/1733400869564/Put/seqid=0 2024-12-05T12:14:31,756 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:31,756 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/baf2c64352dd44578e019505b33ee5ae is 50, key is test_row_0/B:col10/1733400869564/Put/seqid=0 2024-12-05T12:14:31,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742097_1273 (size=12241) 2024-12-05T12:14:31,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742098_1274 (size=12241) 2024-12-05T12:14:31,803 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/130aa228e2a94a858f69313b77c73cd7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/130aa228e2a94a858f69313b77c73cd7 2024-12-05T12:14:31,821 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/baf2c64352dd44578e019505b33ee5ae as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/baf2c64352dd44578e019505b33ee5ae 2024-12-05T12:14:31,826 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/A of 1509b956c8790d96ce37e7755c7bf6f2 into 130aa228e2a94a858f69313b77c73cd7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:31,826 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:31,826 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/A, priority=12, startTime=1733400871739; duration=0sec 2024-12-05T12:14:31,826 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:31,826 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:A 2024-12-05T12:14:31,826 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:14:31,827 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/B of 1509b956c8790d96ce37e7755c7bf6f2 into baf2c64352dd44578e019505b33ee5ae(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:31,827 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:31,827 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/B, priority=12, startTime=1733400871739; duration=0sec 2024-12-05T12:14:31,827 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:31,827 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:B 2024-12-05T12:14:31,831 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:14:31,832 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/C is initiating minor compaction (all files) 2024-12-05T12:14:31,832 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/C in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,832 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/d14fdfab8fd74efb917fe458d54c3700, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a350d7f6dab8466491b726253c43c756, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1d7d93d9aa3a4705826ad34ab744bc39, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/741fa51380fc4a1898aea9ac930cb704] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=47.0 K 2024-12-05T12:14:31,832 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting d14fdfab8fd74efb917fe458d54c3700, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733400867609 2024-12-05T12:14:31,832 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting a350d7f6dab8466491b726253c43c756, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733400868895 2024-12-05T12:14:31,833 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d7d93d9aa3a4705826ad34ab744bc39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733400868917 2024-12-05T12:14:31,833 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 741fa51380fc4a1898aea9ac930cb704, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400869564 2024-12-05T12:14:31,844 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#C#compaction#227 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:31,844 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/42008020b0674c01af2867b751a0fa75 is 50, key is test_row_0/C:col10/1733400869564/Put/seqid=0 2024-12-05T12:14:31,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742099_1275 (size=12241) 2024-12-05T12:14:31,870 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/42008020b0674c01af2867b751a0fa75 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/42008020b0674c01af2867b751a0fa75 2024-12-05T12:14:31,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T12:14:31,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:31,878 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T12:14:31,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:31,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6f9a7f4247f641e0a02189b3faac0d1a is 50, key is test_row_0/A:col10/1733400870754/Put/seqid=0 2024-12-05T12:14:31,890 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/C of 1509b956c8790d96ce37e7755c7bf6f2 into 42008020b0674c01af2867b751a0fa75(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:31,890 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:31,890 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/C, priority=12, startTime=1733400871739; duration=0sec 2024-12-05T12:14:31,891 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:31,891 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:C 2024-12-05T12:14:31,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742100_1276 (size=12001) 2024-12-05T12:14:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:31,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:31,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400931923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400931924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400931927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400931927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400931929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400932032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400932043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400932043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400932043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400932047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400932243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T12:14:32,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400932253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400932253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400932253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400932253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,309 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6f9a7f4247f641e0a02189b3faac0d1a 2024-12-05T12:14:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/cd16608eeaf04c82ae6d7fce988eb36e is 50, key is test_row_0/B:col10/1733400870754/Put/seqid=0 2024-12-05T12:14:32,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742101_1277 (size=12001) 2024-12-05T12:14:32,323 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/cd16608eeaf04c82ae6d7fce988eb36e 2024-12-05T12:14:32,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/a5b4c314b28d4309a8480b57f4d90e6a is 50, key is test_row_0/C:col10/1733400870754/Put/seqid=0 2024-12-05T12:14:32,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742102_1278 (size=12001) 2024-12-05T12:14:32,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400932553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400932563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400932563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400932564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400932564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:32,745 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/a5b4c314b28d4309a8480b57f4d90e6a 2024-12-05T12:14:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6f9a7f4247f641e0a02189b3faac0d1a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6f9a7f4247f641e0a02189b3faac0d1a 2024-12-05T12:14:32,756 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6f9a7f4247f641e0a02189b3faac0d1a, entries=150, sequenceid=130, filesize=11.7 K 2024-12-05T12:14:32,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/cd16608eeaf04c82ae6d7fce988eb36e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cd16608eeaf04c82ae6d7fce988eb36e 2024-12-05T12:14:32,763 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cd16608eeaf04c82ae6d7fce988eb36e, entries=150, sequenceid=130, filesize=11.7 K 2024-12-05T12:14:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/a5b4c314b28d4309a8480b57f4d90e6a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a5b4c314b28d4309a8480b57f4d90e6a 2024-12-05T12:14:32,780 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a5b4c314b28d4309a8480b57f4d90e6a, entries=150, sequenceid=130, filesize=11.7 K 2024-12-05T12:14:32,781 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1509b956c8790d96ce37e7755c7bf6f2 in 903ms, sequenceid=130, compaction requested=false 2024-12-05T12:14:32,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:32,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:32,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-05T12:14:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-05T12:14:32,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-05T12:14:32,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6840 sec 2024-12-05T12:14:32,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6950 sec 2024-12-05T12:14:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:33,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T12:14:33,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:33,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/b8454dc2bec04d07947e49a44ffaeb41 is 50, key is test_row_0/A:col10/1733400873064/Put/seqid=0 2024-12-05T12:14:33,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400933087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400933088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400933089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400933091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400933096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742103_1279 (size=12151) 2024-12-05T12:14:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400933203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400933204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400933205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400933206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400933215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T12:14:33,267 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:33,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:33,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-05T12:14:33,275 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T12:14:33,278 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:33,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T12:14:33,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400933413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400933413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400933413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400933413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400933425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T12:14:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:33,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/b8454dc2bec04d07947e49a44ffaeb41 2024-12-05T12:14:33,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/9fb47e9865cb47eba82528216c2b96a6 is 50, key is test_row_0/B:col10/1733400873064/Put/seqid=0 2024-12-05T12:14:33,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742104_1280 (size=12151) 2024-12-05T12:14:33,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/9fb47e9865cb47eba82528216c2b96a6 2024-12-05T12:14:33,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/9a2a0598979c46c886d866a5d61e0b50 is 50, key is test_row_0/C:col10/1733400873064/Put/seqid=0 2024-12-05T12:14:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742105_1281 (size=12151) 2024-12-05T12:14:33,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T12:14:33,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:33,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:33,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:33,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T12:14:33,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400933722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400933722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400933733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400933733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400933733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:33,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T12:14:33,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:33,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:33,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:33,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T12:14:33,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T12:14:33,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:33,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:33,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:33,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:33,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/9a2a0598979c46c886d866a5d61e0b50 2024-12-05T12:14:33,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/b8454dc2bec04d07947e49a44ffaeb41 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b8454dc2bec04d07947e49a44ffaeb41 2024-12-05T12:14:33,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b8454dc2bec04d07947e49a44ffaeb41, entries=150, sequenceid=148, filesize=11.9 K 2024-12-05T12:14:33,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/9fb47e9865cb47eba82528216c2b96a6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9fb47e9865cb47eba82528216c2b96a6 2024-12-05T12:14:33,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9fb47e9865cb47eba82528216c2b96a6, entries=150, sequenceid=148, filesize=11.9 K 2024-12-05T12:14:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/9a2a0598979c46c886d866a5d61e0b50 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9a2a0598979c46c886d866a5d61e0b50 2024-12-05T12:14:33,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9a2a0598979c46c886d866a5d61e0b50, entries=150, sequenceid=148, filesize=11.9 K 2024-12-05T12:14:33,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1509b956c8790d96ce37e7755c7bf6f2 in 909ms, sequenceid=148, compaction requested=true 2024-12-05T12:14:33,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:33,974 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:33,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:33,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:33,975 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:33,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:33,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:33,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:33,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:33,975 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:33,976 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/A is initiating minor compaction (all files) 2024-12-05T12:14:33,976 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:33,976 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/B is initiating minor compaction (all files) 2024-12-05T12:14:33,976 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/A in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:33,976 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/B in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:33,976 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/130aa228e2a94a858f69313b77c73cd7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6f9a7f4247f641e0a02189b3faac0d1a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b8454dc2bec04d07947e49a44ffaeb41] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=35.5 K 2024-12-05T12:14:33,976 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/baf2c64352dd44578e019505b33ee5ae, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cd16608eeaf04c82ae6d7fce988eb36e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9fb47e9865cb47eba82528216c2b96a6] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=35.5 K 2024-12-05T12:14:33,976 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 130aa228e2a94a858f69313b77c73cd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400869564 2024-12-05T12:14:33,976 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting baf2c64352dd44578e019505b33ee5ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400869564 2024-12-05T12:14:33,976 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd16608eeaf04c82ae6d7fce988eb36e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733400870739 2024-12-05T12:14:33,977 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f9a7f4247f641e0a02189b3faac0d1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733400870739 2024-12-05T12:14:33,977 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fb47e9865cb47eba82528216c2b96a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733400871925 2024-12-05T12:14:33,977 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b8454dc2bec04d07947e49a44ffaeb41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733400871925 2024-12-05T12:14:33,990 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:33,991 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/2dbbd74ec2ec489884b495bbb5d12e33 is 50, key is test_row_0/A:col10/1733400873064/Put/seqid=0 2024-12-05T12:14:33,993 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:33,994 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/fdc349a836c24970ae56864399c7a907 is 50, key is test_row_0/B:col10/1733400873064/Put/seqid=0 2024-12-05T12:14:33,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742106_1282 (size=12493) 2024-12-05T12:14:34,010 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/2dbbd74ec2ec489884b495bbb5d12e33 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/2dbbd74ec2ec489884b495bbb5d12e33 2024-12-05T12:14:34,026 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/A of 1509b956c8790d96ce37e7755c7bf6f2 into 2dbbd74ec2ec489884b495bbb5d12e33(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:34,026 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:34,026 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/A, priority=13, startTime=1733400873974; duration=0sec 2024-12-05T12:14:34,026 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:34,026 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:A 2024-12-05T12:14:34,026 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:34,028 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:34,028 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/C is initiating minor compaction (all files) 2024-12-05T12:14:34,028 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/C in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:34,029 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/42008020b0674c01af2867b751a0fa75, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a5b4c314b28d4309a8480b57f4d90e6a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9a2a0598979c46c886d866a5d61e0b50] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=35.5 K 2024-12-05T12:14:34,029 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 42008020b0674c01af2867b751a0fa75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733400869564 2024-12-05T12:14:34,030 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting a5b4c314b28d4309a8480b57f4d90e6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733400870739 2024-12-05T12:14:34,030 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a2a0598979c46c886d866a5d61e0b50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733400871925 2024-12-05T12:14:34,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742107_1283 (size=12493) 2024-12-05T12:14:34,048 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/fdc349a836c24970ae56864399c7a907 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/fdc349a836c24970ae56864399c7a907 2024-12-05T12:14:34,060 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/B of 1509b956c8790d96ce37e7755c7bf6f2 into fdc349a836c24970ae56864399c7a907(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:34,060 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:34,060 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/B, priority=13, startTime=1733400873975; duration=0sec 2024-12-05T12:14:34,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T12:14:34,060 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:34,060 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:B 2024-12-05T12:14:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:34,061 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T12:14:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:34,065 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#C#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:34,066 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/03d4c3a4562944dbab11063e3e69fd34 is 50, key is test_row_0/C:col10/1733400873064/Put/seqid=0 2024-12-05T12:14:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/88987a1a02e14fa8a7e9d1cd94027453 is 50, key is test_row_0/A:col10/1733400873088/Put/seqid=0 2024-12-05T12:14:34,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742108_1284 (size=12493) 2024-12-05T12:14:34,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742109_1285 (size=12151) 2024-12-05T12:14:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:34,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:34,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400934253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400934255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400934255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400934258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400934257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400934363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400934363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400934363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400934374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400934374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T12:14:34,543 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/88987a1a02e14fa8a7e9d1cd94027453 2024-12-05T12:14:34,543 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/03d4c3a4562944dbab11063e3e69fd34 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/03d4c3a4562944dbab11063e3e69fd34 2024-12-05T12:14:34,552 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/C of 1509b956c8790d96ce37e7755c7bf6f2 into 03d4c3a4562944dbab11063e3e69fd34(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:34,552 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:34,552 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/C, priority=13, startTime=1733400873975; duration=0sec 2024-12-05T12:14:34,552 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:34,552 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:C 2024-12-05T12:14:34,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/9f35bf25f3e448339e1a472fda58559a is 50, key is test_row_0/B:col10/1733400873088/Put/seqid=0 2024-12-05T12:14:34,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400934572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400934573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400934575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400934583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400934587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742110_1286 (size=12151) 2024-12-05T12:14:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400934883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400934883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400934883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400934893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400934902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:35,008 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/9f35bf25f3e448339e1a472fda58559a 2024-12-05T12:14:35,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/64a358fd323a40aa8411164b92b75635 is 50, key is test_row_0/C:col10/1733400873088/Put/seqid=0 2024-12-05T12:14:35,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742111_1287 (size=12151) 2024-12-05T12:14:35,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400935393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:35,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400935394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:35,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:35,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:35,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:35,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:35,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400935399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:35,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400935403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:35,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400935413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T12:14:35,430 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/64a358fd323a40aa8411164b92b75635 2024-12-05T12:14:35,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/88987a1a02e14fa8a7e9d1cd94027453 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/88987a1a02e14fa8a7e9d1cd94027453 2024-12-05T12:14:35,454 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/88987a1a02e14fa8a7e9d1cd94027453, entries=150, sequenceid=165, filesize=11.9 K 2024-12-05T12:14:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/9f35bf25f3e448339e1a472fda58559a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9f35bf25f3e448339e1a472fda58559a 2024-12-05T12:14:35,468 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9f35bf25f3e448339e1a472fda58559a, entries=150, sequenceid=165, filesize=11.9 K 2024-12-05T12:14:35,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/64a358fd323a40aa8411164b92b75635 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/64a358fd323a40aa8411164b92b75635 2024-12-05T12:14:35,481 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/64a358fd323a40aa8411164b92b75635, entries=150, sequenceid=165, filesize=11.9 K 2024-12-05T12:14:35,482 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1509b956c8790d96ce37e7755c7bf6f2 in 1421ms, sequenceid=165, compaction requested=false 2024-12-05T12:14:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-05T12:14:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-05T12:14:35,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-05T12:14:35,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2070 sec 2024-12-05T12:14:35,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2170 sec 2024-12-05T12:14:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:36,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T12:14:36,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:36,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:36,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:36,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:36,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:36,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:36,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/8989b2c62eb34a6f8ed8514de0119fd1 is 50, key is test_row_0/A:col10/1733400874254/Put/seqid=0 2024-12-05T12:14:36,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400936433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742112_1288 (size=14541) 2024-12-05T12:14:36,443 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 1509b956c8790d96ce37e7755c7bf6f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400936437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400936439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400936439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400936440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400936554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400936554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400936563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400936563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400936763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400936763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400936773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400936773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:36,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:36,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:36,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/8989b2c62eb34a6f8ed8514de0119fd1 2024-12-05T12:14:36,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/134b6087328e46dfa2cb8d1bc394e6cc is 50, key is test_row_0/B:col10/1733400874254/Put/seqid=0 2024-12-05T12:14:36,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742113_1289 (size=12151) 2024-12-05T12:14:37,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400937073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400937073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:37,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400937083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:37,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400937089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:37,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/134b6087328e46dfa2cb8d1bc394e6cc 2024-12-05T12:14:37,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/0cb3db84d0574e238eccfc93fff2c4e3 is 50, key is test_row_0/C:col10/1733400874254/Put/seqid=0 2024-12-05T12:14:37,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742114_1290 (size=12151) 2024-12-05T12:14:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T12:14:37,443 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:37,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-05T12:14:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T12:14:37,455 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:37,456 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:37,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T12:14:37,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400937583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400937584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:37,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400937593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:37,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:37,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400937603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:37,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T12:14:37,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:37,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:37,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:37,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:37,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:37,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/0cb3db84d0574e238eccfc93fff2c4e3 2024-12-05T12:14:37,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/8989b2c62eb34a6f8ed8514de0119fd1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/8989b2c62eb34a6f8ed8514de0119fd1 2024-12-05T12:14:37,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/8989b2c62eb34a6f8ed8514de0119fd1, entries=200, sequenceid=184, filesize=14.2 K 2024-12-05T12:14:37,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/134b6087328e46dfa2cb8d1bc394e6cc as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/134b6087328e46dfa2cb8d1bc394e6cc 2024-12-05T12:14:37,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/134b6087328e46dfa2cb8d1bc394e6cc, entries=150, sequenceid=184, filesize=11.9 K 2024-12-05T12:14:37,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/0cb3db84d0574e238eccfc93fff2c4e3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0cb3db84d0574e238eccfc93fff2c4e3 2024-12-05T12:14:37,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0cb3db84d0574e238eccfc93fff2c4e3, entries=150, sequenceid=184, filesize=11.9 K 2024-12-05T12:14:37,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 1509b956c8790d96ce37e7755c7bf6f2 in 1286ms, sequenceid=184, compaction requested=true 2024-12-05T12:14:37,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:37,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:37,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:37,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:37,692 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:37,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:37,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:37,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T12:14:37,692 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:37,693 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/A is initiating minor compaction (all files) 2024-12-05T12:14:37,693 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/A in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:37,693 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/2dbbd74ec2ec489884b495bbb5d12e33, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/88987a1a02e14fa8a7e9d1cd94027453, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/8989b2c62eb34a6f8ed8514de0119fd1] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=38.3 K 2024-12-05T12:14:37,693 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dbbd74ec2ec489884b495bbb5d12e33, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733400871925 2024-12-05T12:14:37,694 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88987a1a02e14fa8a7e9d1cd94027453, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733400873086 2024-12-05T12:14:37,694 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:37,695 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8989b2c62eb34a6f8ed8514de0119fd1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400874252 2024-12-05T12:14:37,695 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:37,695 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/B is initiating minor compaction (all files) 2024-12-05T12:14:37,695 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/B in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:37,695 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/fdc349a836c24970ae56864399c7a907, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9f35bf25f3e448339e1a472fda58559a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/134b6087328e46dfa2cb8d1bc394e6cc] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=35.9 K 2024-12-05T12:14:37,696 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc349a836c24970ae56864399c7a907, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733400871925 2024-12-05T12:14:37,696 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f35bf25f3e448339e1a472fda58559a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733400873086 2024-12-05T12:14:37,697 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 134b6087328e46dfa2cb8d1bc394e6cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400874252 2024-12-05T12:14:37,703 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:37,704 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/641451da50cc461a8f7ec34e81f4ec56 is 50, key is test_row_0/A:col10/1733400874254/Put/seqid=0 2024-12-05T12:14:37,705 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:37,705 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/d04f37ffd0ff44c7b0d23999a397c7fd is 50, key is test_row_0/B:col10/1733400874254/Put/seqid=0 2024-12-05T12:14:37,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742115_1291 (size=12595) 2024-12-05T12:14:37,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742116_1292 (size=12595) 2024-12-05T12:14:37,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T12:14:37,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:37,763 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T12:14:37,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:37,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:37,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:37,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:37,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:37,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:37,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6dd3094dc1fb4906be63fbeb56a24b8c is 50, key is test_row_0/A:col10/1733400876437/Put/seqid=0 2024-12-05T12:14:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T12:14:37,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742117_1293 (size=12151) 2024-12-05T12:14:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T12:14:38,121 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/641451da50cc461a8f7ec34e81f4ec56 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/641451da50cc461a8f7ec34e81f4ec56 2024-12-05T12:14:38,128 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/A of 1509b956c8790d96ce37e7755c7bf6f2 into 641451da50cc461a8f7ec34e81f4ec56(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:38,128 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:38,128 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/A, priority=13, startTime=1733400877691; duration=0sec 2024-12-05T12:14:38,129 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:38,129 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:A 2024-12-05T12:14:38,129 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:38,130 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:38,130 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/C is initiating minor compaction (all files) 2024-12-05T12:14:38,130 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/C in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:38,130 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/03d4c3a4562944dbab11063e3e69fd34, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/64a358fd323a40aa8411164b92b75635, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0cb3db84d0574e238eccfc93fff2c4e3] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=35.9 K 2024-12-05T12:14:38,131 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03d4c3a4562944dbab11063e3e69fd34, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733400871925 2024-12-05T12:14:38,131 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64a358fd323a40aa8411164b92b75635, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733400873086 2024-12-05T12:14:38,131 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cb3db84d0574e238eccfc93fff2c4e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400874252 2024-12-05T12:14:38,131 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/d04f37ffd0ff44c7b0d23999a397c7fd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d04f37ffd0ff44c7b0d23999a397c7fd 2024-12-05T12:14:38,138 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/B of 1509b956c8790d96ce37e7755c7bf6f2 into d04f37ffd0ff44c7b0d23999a397c7fd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:38,138 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:38,138 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/B, priority=13, startTime=1733400877692; duration=0sec 2024-12-05T12:14:38,139 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:38,139 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:B 2024-12-05T12:14:38,142 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:38,142 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/6a7a9287a976427c8f6bc3b4a6eb97d8 is 50, key is test_row_0/C:col10/1733400874254/Put/seqid=0 2024-12-05T12:14:38,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742118_1294 (size=12595) 2024-12-05T12:14:38,154 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/6a7a9287a976427c8f6bc3b4a6eb97d8 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/6a7a9287a976427c8f6bc3b4a6eb97d8 2024-12-05T12:14:38,161 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/C of 1509b956c8790d96ce37e7755c7bf6f2 into 6a7a9287a976427c8f6bc3b4a6eb97d8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:38,162 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:38,162 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/C, priority=13, startTime=1733400877692; duration=0sec 2024-12-05T12:14:38,162 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:38,162 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:C 2024-12-05T12:14:38,175 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6dd3094dc1fb4906be63fbeb56a24b8c 2024-12-05T12:14:38,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/593944b851374b24ae678a73ddb07ddf is 50, key is test_row_0/B:col10/1733400876437/Put/seqid=0 2024-12-05T12:14:38,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742119_1295 (size=12151) 2024-12-05T12:14:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:38,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:38,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400938526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:38,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T12:14:38,595 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/593944b851374b24ae678a73ddb07ddf 2024-12-05T12:14:38,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/bbfa3465abe54c9c928ea6123e03d4a5 is 50, key is test_row_0/C:col10/1733400876437/Put/seqid=0 2024-12-05T12:14:38,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400938603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:38,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400938603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:38,605 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 1509b956c8790d96ce37e7755c7bf6f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:38,606 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 1509b956c8790d96ce37e7755c7bf6f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:38,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742120_1296 (size=12151) 2024-12-05T12:14:38,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400938613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:38,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400938613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:38,615 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 1509b956c8790d96ce37e7755c7bf6f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:38,615 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in 1509b956c8790d96ce37e7755c7bf6f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400938632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:38,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400938844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:39,013 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/bbfa3465abe54c9c928ea6123e03d4a5 2024-12-05T12:14:39,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6dd3094dc1fb4906be63fbeb56a24b8c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6dd3094dc1fb4906be63fbeb56a24b8c 2024-12-05T12:14:39,071 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6dd3094dc1fb4906be63fbeb56a24b8c, entries=150, sequenceid=199, filesize=11.9 K 2024-12-05T12:14:39,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/593944b851374b24ae678a73ddb07ddf as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/593944b851374b24ae678a73ddb07ddf 2024-12-05T12:14:39,096 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/593944b851374b24ae678a73ddb07ddf, entries=150, sequenceid=199, filesize=11.9 K 2024-12-05T12:14:39,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/bbfa3465abe54c9c928ea6123e03d4a5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/bbfa3465abe54c9c928ea6123e03d4a5 2024-12-05T12:14:39,106 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/bbfa3465abe54c9c928ea6123e03d4a5, entries=150, sequenceid=199, filesize=11.9 K 2024-12-05T12:14:39,109 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1509b956c8790d96ce37e7755c7bf6f2 in 1346ms, sequenceid=199, compaction requested=false 2024-12-05T12:14:39,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:39,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:39,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-05T12:14:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-05T12:14:39,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-05T12:14:39,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6550 sec 2024-12-05T12:14:39,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6750 sec 2024-12-05T12:14:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:39,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T12:14:39,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:39,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/58c111355a6947c08a07102258f1af6b is 50, key is test_row_0/A:col10/1733400879154/Put/seqid=0 2024-12-05T12:14:39,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742121_1297 (size=14541) 2024-12-05T12:14:39,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/58c111355a6947c08a07102258f1af6b 2024-12-05T12:14:39,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/43be3ee3b4c543f8a47eeae473ae00b6 is 50, key is test_row_0/B:col10/1733400879154/Put/seqid=0 2024-12-05T12:14:39,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400939251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:39,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:39,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742122_1298 (size=12151) 2024-12-05T12:14:39,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/43be3ee3b4c543f8a47eeae473ae00b6 2024-12-05T12:14:39,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/1c1d7be279df469983406291504af839 is 50, key is test_row_0/C:col10/1733400879154/Put/seqid=0 2024-12-05T12:14:39,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742123_1299 (size=12151) 2024-12-05T12:14:39,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/1c1d7be279df469983406291504af839 2024-12-05T12:14:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/58c111355a6947c08a07102258f1af6b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/58c111355a6947c08a07102258f1af6b 2024-12-05T12:14:39,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400939363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:39,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/58c111355a6947c08a07102258f1af6b, entries=200, sequenceid=220, filesize=14.2 K 2024-12-05T12:14:39,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/43be3ee3b4c543f8a47eeae473ae00b6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/43be3ee3b4c543f8a47eeae473ae00b6 2024-12-05T12:14:39,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/43be3ee3b4c543f8a47eeae473ae00b6, entries=150, sequenceid=220, filesize=11.9 K 2024-12-05T12:14:39,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/1c1d7be279df469983406291504af839 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1c1d7be279df469983406291504af839 2024-12-05T12:14:39,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1c1d7be279df469983406291504af839, entries=150, sequenceid=220, filesize=11.9 K 2024-12-05T12:14:39,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1509b956c8790d96ce37e7755c7bf6f2 in 239ms, sequenceid=220, compaction requested=true 2024-12-05T12:14:39,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:39,398 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:39,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:39,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:39,398 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:39,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:39,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:39,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:39,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:39,400 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:39,400 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:39,400 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/A is initiating minor compaction (all files) 2024-12-05T12:14:39,400 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/B is initiating minor compaction (all files) 2024-12-05T12:14:39,400 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/A in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:39,400 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/B in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:39,400 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/641451da50cc461a8f7ec34e81f4ec56, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6dd3094dc1fb4906be63fbeb56a24b8c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/58c111355a6947c08a07102258f1af6b] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=38.4 K 2024-12-05T12:14:39,400 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d04f37ffd0ff44c7b0d23999a397c7fd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/593944b851374b24ae678a73ddb07ddf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/43be3ee3b4c543f8a47eeae473ae00b6] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=36.0 K 2024-12-05T12:14:39,401 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 641451da50cc461a8f7ec34e81f4ec56, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400874252 2024-12-05T12:14:39,401 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d04f37ffd0ff44c7b0d23999a397c7fd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400874252 2024-12-05T12:14:39,401 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 593944b851374b24ae678a73ddb07ddf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733400876432 2024-12-05T12:14:39,401 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dd3094dc1fb4906be63fbeb56a24b8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733400876432 2024-12-05T12:14:39,401 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 43be3ee3b4c543f8a47eeae473ae00b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733400878511 2024-12-05T12:14:39,401 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58c111355a6947c08a07102258f1af6b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733400878511 2024-12-05T12:14:39,412 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:39,413 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/77bca7672c8b404ca9b13eb86e2f5481 is 50, key is test_row_0/B:col10/1733400879154/Put/seqid=0 2024-12-05T12:14:39,416 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:39,417 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/170740ff09b9484cb177e5490fe560a5 is 50, key is test_row_0/A:col10/1733400879154/Put/seqid=0 2024-12-05T12:14:39,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742124_1300 (size=12697) 2024-12-05T12:14:39,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742125_1301 (size=12697) 2024-12-05T12:14:39,444 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/77bca7672c8b404ca9b13eb86e2f5481 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77bca7672c8b404ca9b13eb86e2f5481 2024-12-05T12:14:39,444 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/170740ff09b9484cb177e5490fe560a5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/170740ff09b9484cb177e5490fe560a5 2024-12-05T12:14:39,452 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/A of 1509b956c8790d96ce37e7755c7bf6f2 into 170740ff09b9484cb177e5490fe560a5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:39,452 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:39,452 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/A, priority=13, startTime=1733400879398; duration=0sec 2024-12-05T12:14:39,452 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:39,452 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:A 2024-12-05T12:14:39,452 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:39,454 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/B of 1509b956c8790d96ce37e7755c7bf6f2 into 77bca7672c8b404ca9b13eb86e2f5481(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:39,454 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:39,454 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/B, priority=13, startTime=1733400879398; duration=0sec 2024-12-05T12:14:39,454 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:39,454 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/C is initiating minor compaction (all files) 2024-12-05T12:14:39,454 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/C in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:39,455 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:39,455 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:B 2024-12-05T12:14:39,455 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/6a7a9287a976427c8f6bc3b4a6eb97d8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/bbfa3465abe54c9c928ea6123e03d4a5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1c1d7be279df469983406291504af839] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=36.0 K 2024-12-05T12:14:39,455 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a7a9287a976427c8f6bc3b4a6eb97d8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400874252 2024-12-05T12:14:39,456 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbfa3465abe54c9c928ea6123e03d4a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733400876432 2024-12-05T12:14:39,456 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c1d7be279df469983406291504af839, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733400878511 2024-12-05T12:14:39,477 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:39,477 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/9276253b2e8c4dac88857cbcd76b9884 is 50, key is test_row_0/C:col10/1733400879154/Put/seqid=0 2024-12-05T12:14:39,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742126_1302 (size=12697) 2024-12-05T12:14:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:39,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T12:14:39,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:39,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:39,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:39,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:39,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:39,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:39,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6b5770520ae949aebd8c43698259faaf is 50, key is test_row_0/A:col10/1733400879573/Put/seqid=0 2024-12-05T12:14:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T12:14:39,603 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:39,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-05T12:14:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T12:14:39,610 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:39,611 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:39,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:39,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742127_1303 (size=12151) 2024-12-05T12:14:39,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400939669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T12:14:39,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T12:14:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:39,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:39,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400939782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:39,898 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/9276253b2e8c4dac88857cbcd76b9884 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9276253b2e8c4dac88857cbcd76b9884 2024-12-05T12:14:39,915 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/C of 1509b956c8790d96ce37e7755c7bf6f2 into 9276253b2e8c4dac88857cbcd76b9884(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:39,915 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:39,915 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/C, priority=13, startTime=1733400879399; duration=0sec 2024-12-05T12:14:39,915 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:39,915 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:C 2024-12-05T12:14:39,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T12:14:39,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:39,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:39,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:39,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T12:14:39,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400939993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:40,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6b5770520ae949aebd8c43698259faaf 2024-12-05T12:14:40,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/229364dfdd2545339d6f4e5abec9ab73 is 50, key is test_row_0/B:col10/1733400879573/Put/seqid=0 2024-12-05T12:14:40,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742128_1304 (size=12151) 2024-12-05T12:14:40,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T12:14:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T12:14:40,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:40,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T12:14:40,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400940303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:40,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T12:14:40,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:40,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/229364dfdd2545339d6f4e5abec9ab73 2024-12-05T12:14:40,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/ddb272996a9c4a02be111304cedc1ebf is 50, key is test_row_0/C:col10/1733400879573/Put/seqid=0 2024-12-05T12:14:40,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742129_1305 (size=12151) 2024-12-05T12:14:40,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T12:14:40,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:40,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400940612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:40,615 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 1509b956c8790d96ce37e7755c7bf6f2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:40,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400940622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:40,625 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 1509b956c8790d96ce37e7755c7bf6f2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:40,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400940623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:40,626 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in 1509b956c8790d96ce37e7755c7bf6f2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:40,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:40,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400940643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:40,646 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 1509b956c8790d96ce37e7755c7bf6f2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:40,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:40,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T12:14:40,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T12:14:40,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400940812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:40,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T12:14:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:40,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:40,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/ddb272996a9c4a02be111304cedc1ebf 2024-12-05T12:14:40,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6b5770520ae949aebd8c43698259faaf as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6b5770520ae949aebd8c43698259faaf 2024-12-05T12:14:40,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6b5770520ae949aebd8c43698259faaf, entries=150, sequenceid=237, filesize=11.9 K 2024-12-05T12:14:40,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/229364dfdd2545339d6f4e5abec9ab73 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/229364dfdd2545339d6f4e5abec9ab73 2024-12-05T12:14:40,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/229364dfdd2545339d6f4e5abec9ab73, entries=150, sequenceid=237, filesize=11.9 K 2024-12-05T12:14:40,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/ddb272996a9c4a02be111304cedc1ebf as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ddb272996a9c4a02be111304cedc1ebf 2024-12-05T12:14:40,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ddb272996a9c4a02be111304cedc1ebf, entries=150, sequenceid=237, filesize=11.9 K 2024-12-05T12:14:40,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1509b956c8790d96ce37e7755c7bf6f2 in 1332ms, sequenceid=237, compaction requested=false 2024-12-05T12:14:40,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:41,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T12:14:41,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:41,010 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T12:14:41,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:41,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:41,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/38c2ad9cad21452582a174848c8aeac9 is 50, key is test_row_0/A:col10/1733400879666/Put/seqid=0 2024-12-05T12:14:41,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742130_1306 (size=12151) 2024-12-05T12:14:41,027 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/38c2ad9cad21452582a174848c8aeac9 2024-12-05T12:14:41,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/835dd78f5ea94b7abdc2aa3d0c786db7 is 50, key is test_row_0/B:col10/1733400879666/Put/seqid=0 2024-12-05T12:14:41,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742131_1307 (size=12151) 2024-12-05T12:14:41,108 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/835dd78f5ea94b7abdc2aa3d0c786db7 2024-12-05T12:14:41,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/0a4c20bfc1eb4dde8cba45b18bbb5bcd is 50, key is test_row_0/C:col10/1733400879666/Put/seqid=0 2024-12-05T12:14:41,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742132_1308 (size=12151) 2024-12-05T12:14:41,173 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/0a4c20bfc1eb4dde8cba45b18bbb5bcd 2024-12-05T12:14:41,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/38c2ad9cad21452582a174848c8aeac9 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/38c2ad9cad21452582a174848c8aeac9 2024-12-05T12:14:41,188 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/38c2ad9cad21452582a174848c8aeac9, entries=150, sequenceid=255, filesize=11.9 K 2024-12-05T12:14:41,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/835dd78f5ea94b7abdc2aa3d0c786db7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/835dd78f5ea94b7abdc2aa3d0c786db7 2024-12-05T12:14:41,197 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/835dd78f5ea94b7abdc2aa3d0c786db7, entries=150, sequenceid=255, filesize=11.9 K 2024-12-05T12:14:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/0a4c20bfc1eb4dde8cba45b18bbb5bcd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0a4c20bfc1eb4dde8cba45b18bbb5bcd 2024-12-05T12:14:41,218 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0a4c20bfc1eb4dde8cba45b18bbb5bcd, entries=150, sequenceid=255, filesize=11.9 K 2024-12-05T12:14:41,219 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 1509b956c8790d96ce37e7755c7bf6f2 in 208ms, sequenceid=255, compaction requested=true 2024-12-05T12:14:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-05T12:14:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-05T12:14:41,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-05T12:14:41,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6100 sec 2024-12-05T12:14:41,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6210 sec 2024-12-05T12:14:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T12:14:41,763 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:41,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-05T12:14:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T12:14:41,775 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:41,776 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:41,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:41,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:14:41,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:41,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:41,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:41,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:41,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:41,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:41,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/4ed310385b904296be50e8212c06bbf1 is 50, key is test_row_0/A:col10/1733400881843/Put/seqid=0 2024-12-05T12:14:41,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742133_1309 (size=9857) 2024-12-05T12:14:41,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/4ed310385b904296be50e8212c06bbf1 2024-12-05T12:14:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T12:14:41,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/3b9d05ed9cde473782ec2b453bb48e6b is 50, key is test_row_0/B:col10/1733400881843/Put/seqid=0 2024-12-05T12:14:41,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742134_1310 (size=9857) 2024-12-05T12:14:41,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T12:14:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:41,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:41,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400881889, max timestamp=1733400881955Num uniques 859; 2024-12-05T12:14:41,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400881889, max timestamp=1733400881955Num uniques 859; 2024-12-05T12:14:41,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400881889, max timestamp=1733400881955Num uniques 859; 2024-12-05T12:14:41,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:41,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:41,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:42,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400942020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:42,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:42,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:42,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T12:14:42,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:42,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:42,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:42,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:42,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T12:14:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:42,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400942133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:42,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T12:14:42,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:42,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:42,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:42,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:42,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:42,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/3b9d05ed9cde473782ec2b453bb48e6b 2024-12-05T12:14:42,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/9f01c92fcd0a4e47a7a77d2116ef31ac is 50, key is test_row_0/C:col10/1733400881843/Put/seqid=0 2024-12-05T12:14:42,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742135_1311 (size=9857) 2024-12-05T12:14:42,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/9f01c92fcd0a4e47a7a77d2116ef31ac 2024-12-05T12:14:42,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400942345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:42,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/4ed310385b904296be50e8212c06bbf1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/4ed310385b904296be50e8212c06bbf1 2024-12-05T12:14:42,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/4ed310385b904296be50e8212c06bbf1, entries=100, sequenceid=265, filesize=9.6 K 2024-12-05T12:14:42,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/3b9d05ed9cde473782ec2b453bb48e6b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/3b9d05ed9cde473782ec2b453bb48e6b 2024-12-05T12:14:42,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/3b9d05ed9cde473782ec2b453bb48e6b, entries=100, sequenceid=265, filesize=9.6 K 2024-12-05T12:14:42,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/9f01c92fcd0a4e47a7a77d2116ef31ac as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9f01c92fcd0a4e47a7a77d2116ef31ac 2024-12-05T12:14:42,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9f01c92fcd0a4e47a7a77d2116ef31ac, entries=100, sequenceid=265, filesize=9.6 K 2024-12-05T12:14:42,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1509b956c8790d96ce37e7755c7bf6f2 in 534ms, sequenceid=265, compaction requested=true 2024-12-05T12:14:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:42,381 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:14:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:42,381 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:14:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:42,382 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46856 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:14:42,382 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46856 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:14:42,382 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/B is initiating minor compaction (all files) 2024-12-05T12:14:42,382 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/A is initiating minor compaction (all files) 2024-12-05T12:14:42,382 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/B in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:42,382 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/A in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:42,382 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77bca7672c8b404ca9b13eb86e2f5481, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/229364dfdd2545339d6f4e5abec9ab73, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/835dd78f5ea94b7abdc2aa3d0c786db7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/3b9d05ed9cde473782ec2b453bb48e6b] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=45.8 K 2024-12-05T12:14:42,382 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/170740ff09b9484cb177e5490fe560a5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6b5770520ae949aebd8c43698259faaf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/38c2ad9cad21452582a174848c8aeac9, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/4ed310385b904296be50e8212c06bbf1] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=45.8 K 2024-12-05T12:14:42,382 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 170740ff09b9484cb177e5490fe560a5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733400878511 2024-12-05T12:14:42,382 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 77bca7672c8b404ca9b13eb86e2f5481, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733400878511 2024-12-05T12:14:42,382 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b5770520ae949aebd8c43698259faaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733400879234 2024-12-05T12:14:42,382 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 229364dfdd2545339d6f4e5abec9ab73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733400879234 2024-12-05T12:14:42,383 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c2ad9cad21452582a174848c8aeac9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733400879634 2024-12-05T12:14:42,383 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 835dd78f5ea94b7abdc2aa3d0c786db7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733400879634 2024-12-05T12:14:42,383 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ed310385b904296be50e8212c06bbf1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733400881843 2024-12-05T12:14:42,383 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b9d05ed9cde473782ec2b453bb48e6b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733400881843 2024-12-05T12:14:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T12:14:42,393 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:42,394 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6c96f011c3d449f78bfd7bc5687edc88 is 50, key is test_row_0/A:col10/1733400881843/Put/seqid=0 2024-12-05T12:14:42,397 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#B#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:42,398 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/1379752a72a34e408884dc05aa7d0cf6 is 50, key is test_row_0/B:col10/1733400881843/Put/seqid=0 2024-12-05T12:14:42,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T12:14:42,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:42,403 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T12:14:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:42,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742136_1312 (size=12933) 2024-12-05T12:14:42,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742137_1313 (size=12933) 2024-12-05T12:14:42,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/9abe5443c67d4242a10eb42e8844a262 is 50, key is test_row_0/A:col10/1733400882009/Put/seqid=0 2024-12-05T12:14:42,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742138_1314 (size=12301) 2024-12-05T12:14:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:42,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:42,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400942690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:42,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400942803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:42,824 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/6c96f011c3d449f78bfd7bc5687edc88 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6c96f011c3d449f78bfd7bc5687edc88 2024-12-05T12:14:42,829 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/A of 1509b956c8790d96ce37e7755c7bf6f2 into 6c96f011c3d449f78bfd7bc5687edc88(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:42,829 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:42,829 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/A, priority=12, startTime=1733400882381; duration=0sec 2024-12-05T12:14:42,829 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:42,829 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:A 2024-12-05T12:14:42,829 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:14:42,830 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46856 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:14:42,830 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/C is initiating minor compaction (all files) 2024-12-05T12:14:42,831 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/C in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:42,831 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9276253b2e8c4dac88857cbcd76b9884, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ddb272996a9c4a02be111304cedc1ebf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0a4c20bfc1eb4dde8cba45b18bbb5bcd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9f01c92fcd0a4e47a7a77d2116ef31ac] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=45.8 K 2024-12-05T12:14:42,831 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9276253b2e8c4dac88857cbcd76b9884, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733400878511 2024-12-05T12:14:42,831 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddb272996a9c4a02be111304cedc1ebf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733400879234 2024-12-05T12:14:42,831 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a4c20bfc1eb4dde8cba45b18bbb5bcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733400879634 2024-12-05T12:14:42,832 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f01c92fcd0a4e47a7a77d2116ef31ac, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733400881843 2024-12-05T12:14:42,837 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/1379752a72a34e408884dc05aa7d0cf6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/1379752a72a34e408884dc05aa7d0cf6 2024-12-05T12:14:42,845 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/9abe5443c67d4242a10eb42e8844a262 2024-12-05T12:14:42,846 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/B of 1509b956c8790d96ce37e7755c7bf6f2 into 1379752a72a34e408884dc05aa7d0cf6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:42,846 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:42,846 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/B, priority=12, startTime=1733400882381; duration=0sec 2024-12-05T12:14:42,846 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:42,846 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:B 2024-12-05T12:14:42,850 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:42,851 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/5136d22c7327474495a947989591837b is 50, key is test_row_0/C:col10/1733400881843/Put/seqid=0 2024-12-05T12:14:42,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/e6520512ab5b4892b674d82e540055b5 is 50, key is test_row_0/B:col10/1733400882009/Put/seqid=0 2024-12-05T12:14:42,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742140_1316 (size=12301) 2024-12-05T12:14:42,861 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/e6520512ab5b4892b674d82e540055b5 2024-12-05T12:14:42,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742139_1315 (size=12933) 2024-12-05T12:14:42,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/c9a94777425241eeafaa04045341a6c4 is 50, key is test_row_0/C:col10/1733400882009/Put/seqid=0 2024-12-05T12:14:42,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742141_1317 (size=12301) 2024-12-05T12:14:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T12:14:43,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400943013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:43,279 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/5136d22c7327474495a947989591837b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/5136d22c7327474495a947989591837b 2024-12-05T12:14:43,286 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/C of 1509b956c8790d96ce37e7755c7bf6f2 into 5136d22c7327474495a947989591837b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:43,286 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:43,286 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/C, priority=12, startTime=1733400882381; duration=0sec 2024-12-05T12:14:43,286 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:43,286 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:C 2024-12-05T12:14:43,287 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/c9a94777425241eeafaa04045341a6c4 2024-12-05T12:14:43,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/9abe5443c67d4242a10eb42e8844a262 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/9abe5443c67d4242a10eb42e8844a262 2024-12-05T12:14:43,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/9abe5443c67d4242a10eb42e8844a262, entries=150, sequenceid=302, filesize=12.0 K 2024-12-05T12:14:43,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/e6520512ab5b4892b674d82e540055b5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/e6520512ab5b4892b674d82e540055b5 2024-12-05T12:14:43,306 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/e6520512ab5b4892b674d82e540055b5, entries=150, sequenceid=302, filesize=12.0 K 2024-12-05T12:14:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/c9a94777425241eeafaa04045341a6c4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/c9a94777425241eeafaa04045341a6c4 2024-12-05T12:14:43,317 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/c9a94777425241eeafaa04045341a6c4, entries=150, sequenceid=302, filesize=12.0 K 2024-12-05T12:14:43,319 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 1509b956c8790d96ce37e7755c7bf6f2 in 916ms, sequenceid=302, compaction requested=false 2024-12-05T12:14:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-05T12:14:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-05T12:14:43,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-05T12:14:43,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5450 sec 2024-12-05T12:14:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:43,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T12:14:43,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:43,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:43,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:43,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:43,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:43,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:43,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5640 sec 2024-12-05T12:14:43,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/a612d618d20b4051884c4ebd070d8c3d is 50, key is test_row_0/A:col10/1733400882688/Put/seqid=0 2024-12-05T12:14:43,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742142_1318 (size=14741) 2024-12-05T12:14:43,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400883349, max timestamp=1733400883415Num uniques 859; 2024-12-05T12:14:43,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400883349, max timestamp=1733400883415Num uniques 859; 2024-12-05T12:14:43,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400883349, max timestamp=1733400883415Num uniques 859; 2024-12-05T12:14:43,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:43,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:43,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:14:43,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400943487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:43,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400943603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:43,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/a612d618d20b4051884c4ebd070d8c3d 2024-12-05T12:14:43,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/426b2de8c70a4dba94aa0dc704161cc0 is 50, key is test_row_0/B:col10/1733400882688/Put/seqid=0 2024-12-05T12:14:43,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742143_1319 (size=12301) 2024-12-05T12:14:43,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/426b2de8c70a4dba94aa0dc704161cc0 2024-12-05T12:14:43,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400943813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:43,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/435132c0ed884705965c10c11ef8f2bb is 50, key is test_row_0/C:col10/1733400882688/Put/seqid=0 2024-12-05T12:14:43,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742144_1320 (size=12301) 2024-12-05T12:14:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T12:14:43,913 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:43,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-05T12:14:43,916 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T12:14:43,917 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:43,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T12:14:44,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T12:14:44,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:44,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:44,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:44,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:44,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:44,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400944122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T12:14:44,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:44,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:44,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:44,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:44,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T12:14:44,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/435132c0ed884705965c10c11ef8f2bb 2024-12-05T12:14:44,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/a612d618d20b4051884c4ebd070d8c3d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a612d618d20b4051884c4ebd070d8c3d 2024-12-05T12:14:44,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a612d618d20b4051884c4ebd070d8c3d, entries=200, sequenceid=316, filesize=14.4 K 2024-12-05T12:14:44,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/426b2de8c70a4dba94aa0dc704161cc0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/426b2de8c70a4dba94aa0dc704161cc0 2024-12-05T12:14:44,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/426b2de8c70a4dba94aa0dc704161cc0, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T12:14:44,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/435132c0ed884705965c10c11ef8f2bb as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/435132c0ed884705965c10c11ef8f2bb 2024-12-05T12:14:44,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/435132c0ed884705965c10c11ef8f2bb, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T12:14:44,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1509b956c8790d96ce37e7755c7bf6f2 in 958ms, sequenceid=316, compaction requested=true 2024-12-05T12:14:44,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:44,287 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:44,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:44,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:44,288 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:44,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:44,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:44,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:44,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:44,290 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:44,290 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/A is initiating minor compaction (all files) 2024-12-05T12:14:44,290 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37535 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:44,290 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/A in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:44,290 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/B is initiating minor compaction (all files) 2024-12-05T12:14:44,290 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/B in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:44,290 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6c96f011c3d449f78bfd7bc5687edc88, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/9abe5443c67d4242a10eb42e8844a262, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a612d618d20b4051884c4ebd070d8c3d] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=39.0 K 2024-12-05T12:14:44,290 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/1379752a72a34e408884dc05aa7d0cf6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/e6520512ab5b4892b674d82e540055b5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/426b2de8c70a4dba94aa0dc704161cc0] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=36.7 K 2024-12-05T12:14:44,290 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c96f011c3d449f78bfd7bc5687edc88, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733400879650 2024-12-05T12:14:44,290 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 1379752a72a34e408884dc05aa7d0cf6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733400879650 2024-12-05T12:14:44,291 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9abe5443c67d4242a10eb42e8844a262, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733400881996 2024-12-05T12:14:44,291 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting e6520512ab5b4892b674d82e540055b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733400881996 2024-12-05T12:14:44,291 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting a612d618d20b4051884c4ebd070d8c3d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733400882677 2024-12-05T12:14:44,291 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 426b2de8c70a4dba94aa0dc704161cc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733400882677 2024-12-05T12:14:44,304 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:44,305 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/98f3ffafc77040c4a517c0e1f0cc06b4 is 50, key is test_row_0/B:col10/1733400882688/Put/seqid=0 2024-12-05T12:14:44,306 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:44,306 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/814180362b5e479597206a61978c82fd is 50, key is test_row_0/A:col10/1733400882688/Put/seqid=0 2024-12-05T12:14:44,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742146_1322 (size=13085) 2024-12-05T12:14:44,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742145_1321 (size=13085) 2024-12-05T12:14:44,331 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/814180362b5e479597206a61978c82fd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/814180362b5e479597206a61978c82fd 2024-12-05T12:14:44,336 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/A of 1509b956c8790d96ce37e7755c7bf6f2 into 814180362b5e479597206a61978c82fd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:44,336 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:44,336 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/A, priority=13, startTime=1733400884287; duration=0sec 2024-12-05T12:14:44,336 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:44,336 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:A 2024-12-05T12:14:44,336 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:44,337 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37535 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:44,337 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/C is initiating minor compaction (all files) 2024-12-05T12:14:44,337 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/C in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:44,337 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/5136d22c7327474495a947989591837b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/c9a94777425241eeafaa04045341a6c4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/435132c0ed884705965c10c11ef8f2bb] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=36.7 K 2024-12-05T12:14:44,338 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5136d22c7327474495a947989591837b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733400879650 2024-12-05T12:14:44,338 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9a94777425241eeafaa04045341a6c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733400881996 2024-12-05T12:14:44,338 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 435132c0ed884705965c10c11ef8f2bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733400882677 2024-12-05T12:14:44,357 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#C#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:44,357 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/faf3ece209d14aa9b2cc3374a5209491 is 50, key is test_row_0/C:col10/1733400882688/Put/seqid=0 2024-12-05T12:14:44,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T12:14:44,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:44,383 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-05T12:14:44,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:44,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:44,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:44,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:44,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:44,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:14:44,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742147_1323 (size=13085) 2024-12-05T12:14:44,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/51c4789a8f1644cca2200b43f4ac7424 is 50, key is test_row_0/A:col10/1733400883480/Put/seqid=0 2024-12-05T12:14:44,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742148_1324 (size=12301) 2024-12-05T12:14:44,516 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T12:14:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T12:14:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:44,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:44,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400944646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400944648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400944649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400944650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40302 deadline: 1733400944663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,665 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 1509b956c8790d96ce37e7755c7bf6f2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:14:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,738 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/98f3ffafc77040c4a517c0e1f0cc06b4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/98f3ffafc77040c4a517c0e1f0cc06b4 2024-12-05T12:14:44,749 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/B of 1509b956c8790d96ce37e7755c7bf6f2 into 98f3ffafc77040c4a517c0e1f0cc06b4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:44,749 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:44,749 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/B, priority=13, startTime=1733400884288; duration=0sec 2024-12-05T12:14:44,749 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:44,749 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:B 2024-12-05T12:14:44,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400944763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400944763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400944763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400944763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,795 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/faf3ece209d14aa9b2cc3374a5209491 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/faf3ece209d14aa9b2cc3374a5209491 2024-12-05T12:14:44,802 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/C of 1509b956c8790d96ce37e7755c7bf6f2 into faf3ece209d14aa9b2cc3374a5209491(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:44,802 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:44,802 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/C, priority=13, startTime=1733400884289; duration=0sec 2024-12-05T12:14:44,802 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:44,802 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:C 2024-12-05T12:14:44,809 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/51c4789a8f1644cca2200b43f4ac7424 2024-12-05T12:14:44,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/d6ee062341d14d31964a91252898b778 is 50, key is test_row_0/B:col10/1733400883480/Put/seqid=0 2024-12-05T12:14:44,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742149_1325 (size=12301) 2024-12-05T12:14:44,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400944972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400944972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400944972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:44,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:44,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400944973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:44,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:44,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T12:14:45,237 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/d6ee062341d14d31964a91252898b778 2024-12-05T12:14:45,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/ec03839bb4064a60ae31c276cbcc7c9b is 50, key is test_row_0/C:col10/1733400883480/Put/seqid=0 2024-12-05T12:14:45,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742150_1326 (size=12301) 2024-12-05T12:14:45,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400945283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400945283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400945283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400945283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,664 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/ec03839bb4064a60ae31c276cbcc7c9b 2024-12-05T12:14:45,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/51c4789a8f1644cca2200b43f4ac7424 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/51c4789a8f1644cca2200b43f4ac7424 2024-12-05T12:14:45,678 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/51c4789a8f1644cca2200b43f4ac7424, entries=150, sequenceid=353, filesize=12.0 K 2024-12-05T12:14:45,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/d6ee062341d14d31964a91252898b778 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d6ee062341d14d31964a91252898b778 2024-12-05T12:14:45,683 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d6ee062341d14d31964a91252898b778, entries=150, sequenceid=353, filesize=12.0 K 2024-12-05T12:14:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/ec03839bb4064a60ae31c276cbcc7c9b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ec03839bb4064a60ae31c276cbcc7c9b 2024-12-05T12:14:45,688 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ec03839bb4064a60ae31c276cbcc7c9b, entries=150, sequenceid=353, filesize=12.0 K 2024-12-05T12:14:45,689 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 1509b956c8790d96ce37e7755c7bf6f2 in 1306ms, sequenceid=353, compaction requested=false 2024-12-05T12:14:45,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:45,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:45,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-05T12:14:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-05T12:14:45,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-05T12:14:45,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7730 sec 2024-12-05T12:14:45,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.7780 sec 2024-12-05T12:14:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:45,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T12:14:45,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:45,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:45,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:45,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:45,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:45,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:45,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/aeb1b7f6fc8344c49fa69a1dd03e43e9 is 50, key is test_row_0/A:col10/1733400885796/Put/seqid=0 2024-12-05T12:14:45,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742151_1327 (size=12301) 2024-12-05T12:14:45,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400945828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400945830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400945842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400945847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400945943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400945953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400945953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:45,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400945963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:45,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:45,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T12:14:46,063 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:46,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:14:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-05T12:14:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T12:14:46,073 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:14:46,076 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:14:46,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:14:46,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400946154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:46,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:46,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400946165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:46,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:46,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400946173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:46,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400946173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:46,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T12:14:46,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/aeb1b7f6fc8344c49fa69a1dd03e43e9 2024-12-05T12:14:46,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T12:14:46,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/634db6d6f29e43ee811db4a9da78eda9 is 50, key is test_row_0/B:col10/1733400885796/Put/seqid=0 2024-12-05T12:14:46,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:46,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742152_1328 (size=12301) 2024-12-05T12:14:46,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/634db6d6f29e43ee811db4a9da78eda9 2024-12-05T12:14:46,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/80b424a08cca4302962708229617f26a is 50, key is test_row_0/C:col10/1733400885796/Put/seqid=0 2024-12-05T12:14:46,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742153_1329 (size=12301) 2024-12-05T12:14:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T12:14:46,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T12:14:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40356 deadline: 1733400946472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:46,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40314 deadline: 1733400946483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40282 deadline: 1733400946483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:46,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:14:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40352 deadline: 1733400946484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:14:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:14:46,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T12:14:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T12:14:46,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T12:14:46,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:46,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:14:46,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/80b424a08cca4302962708229617f26a 2024-12-05T12:14:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/aeb1b7f6fc8344c49fa69a1dd03e43e9 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/aeb1b7f6fc8344c49fa69a1dd03e43e9 2024-12-05T12:14:46,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/aeb1b7f6fc8344c49fa69a1dd03e43e9, entries=150, sequenceid=371, filesize=12.0 K 2024-12-05T12:14:46,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/634db6d6f29e43ee811db4a9da78eda9 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/634db6d6f29e43ee811db4a9da78eda9 2024-12-05T12:14:46,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/634db6d6f29e43ee811db4a9da78eda9, entries=150, sequenceid=371, filesize=12.0 K 2024-12-05T12:14:46,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/80b424a08cca4302962708229617f26a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/80b424a08cca4302962708229617f26a 2024-12-05T12:14:46,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/80b424a08cca4302962708229617f26a, entries=150, sequenceid=371, filesize=12.0 K 2024-12-05T12:14:46,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1509b956c8790d96ce37e7755c7bf6f2 in 1031ms, sequenceid=371, compaction requested=true 2024-12-05T12:14:46,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:46,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:14:46,830 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:46,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:46,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:14:46,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:46,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1509b956c8790d96ce37e7755c7bf6f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:14:46,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T12:14:46,830 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:46,832 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:46,832 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/A is initiating minor compaction (all files) 2024-12-05T12:14:46,832 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/A in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,832 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/814180362b5e479597206a61978c82fd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/51c4789a8f1644cca2200b43f4ac7424, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/aeb1b7f6fc8344c49fa69a1dd03e43e9] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=36.8 K 2024-12-05T12:14:46,833 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 814180362b5e479597206a61978c82fd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733400882677 2024-12-05T12:14:46,833 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51c4789a8f1644cca2200b43f4ac7424, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733400883444 2024-12-05T12:14:46,833 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeb1b7f6fc8344c49fa69a1dd03e43e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733400885794 2024-12-05T12:14:46,839 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:46,839 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/B is initiating minor compaction (all files) 2024-12-05T12:14:46,839 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/B in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,839 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/98f3ffafc77040c4a517c0e1f0cc06b4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d6ee062341d14d31964a91252898b778, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/634db6d6f29e43ee811db4a9da78eda9] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=36.8 K 2024-12-05T12:14:46,847 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f3ffafc77040c4a517c0e1f0cc06b4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733400882677 2024-12-05T12:14:46,851 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ee062341d14d31964a91252898b778, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733400883444 2024-12-05T12:14:46,856 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 634db6d6f29e43ee811db4a9da78eda9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733400885794 2024-12-05T12:14:46,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T12:14:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,871 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T12:14:46,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:46,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:46,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:46,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:46,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:46,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:46,880 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:46,881 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/c61af2580f314407aec497ba94fa5b8d is 50, key is test_row_0/A:col10/1733400885796/Put/seqid=0 2024-12-05T12:14:46,885 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:46,886 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/31717e0fc0904fac9fd8a5a7d174ba8c is 50, key is test_row_0/B:col10/1733400885796/Put/seqid=0 2024-12-05T12:14:46,897 INFO [Thread-1175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1175. 2024-12-05T12:14:46,897 DEBUG [Thread-1175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:46,897 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,897 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,897 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-12-05T12:14:46,897 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:46,897 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,897 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:46,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:46,900 INFO [Thread-1183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1183. 2024-12-05T12:14:46,900 DEBUG [Thread-1183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:46,900 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,900 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:46,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/066d53bd0cd74e2f87412fd4ae64d4e2 is 50, key is test_row_0/A:col10/1733400885827/Put/seqid=0 2024-12-05T12:14:46,900 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-12-05T12:14:46,900 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:46,900 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,900 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,901 INFO [Thread-1181 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1181. 2024-12-05T12:14:46,901 DEBUG [Thread-1181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:46,901 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,901 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:46,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:46,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742154_1330 (size=13187) 2024-12-05T12:14:46,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742156_1332 (size=12301) 2024-12-05T12:14:46,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742155_1331 (size=13187) 2024-12-05T12:14:46,913 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/c61af2580f314407aec497ba94fa5b8d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/c61af2580f314407aec497ba94fa5b8d 2024-12-05T12:14:46,917 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/A of 1509b956c8790d96ce37e7755c7bf6f2 into c61af2580f314407aec497ba94fa5b8d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:46,917 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:46,917 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/A, priority=13, startTime=1733400886829; duration=0sec 2024-12-05T12:14:46,917 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:14:46,917 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:A 2024-12-05T12:14:46,917 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:14:46,918 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:14:46,918 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 1509b956c8790d96ce37e7755c7bf6f2/C is initiating minor compaction (all files) 2024-12-05T12:14:46,918 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1509b956c8790d96ce37e7755c7bf6f2/C in TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:46,918 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/faf3ece209d14aa9b2cc3374a5209491, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ec03839bb4064a60ae31c276cbcc7c9b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/80b424a08cca4302962708229617f26a] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp, totalSize=36.8 K 2024-12-05T12:14:46,919 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting faf3ece209d14aa9b2cc3374a5209491, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733400882677 2024-12-05T12:14:46,919 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec03839bb4064a60ae31c276cbcc7c9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733400883444 2024-12-05T12:14:46,919 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b424a08cca4302962708229617f26a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733400885794 2024-12-05T12:14:46,924 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1509b956c8790d96ce37e7755c7bf6f2#C#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:14:46,924 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/2688699ce0c64b979d02a4c2a54f396b is 50, key is test_row_0/C:col10/1733400885796/Put/seqid=0 2024-12-05T12:14:46,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742157_1333 (size=13187) 2024-12-05T12:14:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:46,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. as already flushing 2024-12-05T12:14:46,986 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-12-05T12:14:46,986 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:46,986 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,986 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:46,994 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-12-05T12:14:46,994 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:46,994 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,994 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:46,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:47,004 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-12-05T12:14:47,004 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:47,004 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:47,004 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:47,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:47,014 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-12-05T12:14:47,014 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:47,014 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:47,014 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:47,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T12:14:47,309 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/066d53bd0cd74e2f87412fd4ae64d4e2 2024-12-05T12:14:47,313 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/31717e0fc0904fac9fd8a5a7d174ba8c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/31717e0fc0904fac9fd8a5a7d174ba8c 2024-12-05T12:14:47,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/2c874db717524ef1bc4263f5de6edfcd is 50, key is test_row_0/B:col10/1733400885827/Put/seqid=0 2024-12-05T12:14:47,316 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/B of 1509b956c8790d96ce37e7755c7bf6f2 into 31717e0fc0904fac9fd8a5a7d174ba8c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:47,316 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:47,316 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/B, priority=13, startTime=1733400886830; duration=0sec 2024-12-05T12:14:47,317 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:47,317 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:B 2024-12-05T12:14:47,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742158_1334 (size=12301) 2024-12-05T12:14:47,332 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/2688699ce0c64b979d02a4c2a54f396b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/2688699ce0c64b979d02a4c2a54f396b 2024-12-05T12:14:47,338 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1509b956c8790d96ce37e7755c7bf6f2/C of 1509b956c8790d96ce37e7755c7bf6f2 into 2688699ce0c64b979d02a4c2a54f396b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:14:47,338 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:47,338 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., storeName=1509b956c8790d96ce37e7755c7bf6f2/C, priority=13, startTime=1733400886830; duration=0sec 2024-12-05T12:14:47,338 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:14:47,338 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1509b956c8790d96ce37e7755c7bf6f2:C 2024-12-05T12:14:47,718 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/2c874db717524ef1bc4263f5de6edfcd 2024-12-05T12:14:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/f1243f78e96b417faeeaabb3e8f792e2 is 50, key is test_row_0/C:col10/1733400885827/Put/seqid=0 2024-12-05T12:14:47,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742159_1335 (size=12301) 2024-12-05T12:14:48,132 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/f1243f78e96b417faeeaabb3e8f792e2 2024-12-05T12:14:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/066d53bd0cd74e2f87412fd4ae64d4e2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/066d53bd0cd74e2f87412fd4ae64d4e2 2024-12-05T12:14:48,142 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/066d53bd0cd74e2f87412fd4ae64d4e2, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T12:14:48,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/2c874db717524ef1bc4263f5de6edfcd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/2c874db717524ef1bc4263f5de6edfcd 2024-12-05T12:14:48,147 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/2c874db717524ef1bc4263f5de6edfcd, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T12:14:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/f1243f78e96b417faeeaabb3e8f792e2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/f1243f78e96b417faeeaabb3e8f792e2 2024-12-05T12:14:48,152 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/f1243f78e96b417faeeaabb3e8f792e2, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T12:14:48,152 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for 1509b956c8790d96ce37e7755c7bf6f2 in 1281ms, sequenceid=387, compaction requested=false 2024-12-05T12:14:48,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 1509b956c8790d96ce37e7755c7bf6f2: 2024-12-05T12:14:48,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:48,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-05T12:14:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-05T12:14:48,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-05T12:14:48,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0780 sec 2024-12-05T12:14:48,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.0910 sec 2024-12-05T12:14:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T12:14:48,234 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:54,725 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-12-05T12:14:54,725 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:14:54,725 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:54,725 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:54,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 157 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3605 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3544 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3529 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3574 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3572 2024-12-05T12:14:54,726 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T12:14:54,726 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:14:54,726 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:14:54,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:54,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:54,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:14:54,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T12:14:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T12:14:54,730 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400894730"}]},"ts":"1733400894730"} 2024-12-05T12:14:54,732 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T12:14:54,732 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T12:14:54,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T12:14:54,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1509b956c8790d96ce37e7755c7bf6f2, UNASSIGN}] 2024-12-05T12:14:54,736 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1509b956c8790d96ce37e7755c7bf6f2, UNASSIGN 2024-12-05T12:14:54,737 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=1509b956c8790d96ce37e7755c7bf6f2, regionState=CLOSING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:54,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1509b956c8790d96ce37e7755c7bf6f2, UNASSIGN because future has completed 2024-12-05T12:14:54,739 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T12:14:54,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T12:14:54,891 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 1509b956c8790d96ce37e7755c7bf6f2, disabling compactions & flushes 2024-12-05T12:14:54,892 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. after waiting 0 ms 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:54,892 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 1509b956c8790d96ce37e7755c7bf6f2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=A 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=B 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1509b956c8790d96ce37e7755c7bf6f2, store=C 2024-12-05T12:14:54,892 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:14:54,896 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/02f3c312d04f4bfc91abee8d394a90ae is 50, key is test_row_0/A:col10/1733400894724/Put/seqid=0 2024-12-05T12:14:54,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742160_1336 (size=12301) 2024-12-05T12:14:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T12:14:55,301 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/02f3c312d04f4bfc91abee8d394a90ae 2024-12-05T12:14:55,313 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/25833819309741fd941d3286ac59a5c7 is 50, key is test_row_0/B:col10/1733400894724/Put/seqid=0 2024-12-05T12:14:55,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742161_1337 (size=12301) 2024-12-05T12:14:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T12:14:55,718 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/25833819309741fd941d3286ac59a5c7 2024-12-05T12:14:55,728 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/346d404ba76944cead65cc71bc720ca5 is 50, key is test_row_0/C:col10/1733400894724/Put/seqid=0 2024-12-05T12:14:55,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742162_1338 (size=12301) 2024-12-05T12:14:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T12:14:56,134 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/346d404ba76944cead65cc71bc720ca5 2024-12-05T12:14:56,139 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/A/02f3c312d04f4bfc91abee8d394a90ae as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/02f3c312d04f4bfc91abee8d394a90ae 2024-12-05T12:14:56,144 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/02f3c312d04f4bfc91abee8d394a90ae, entries=150, sequenceid=398, filesize=12.0 K 2024-12-05T12:14:56,144 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/B/25833819309741fd941d3286ac59a5c7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/25833819309741fd941d3286ac59a5c7 2024-12-05T12:14:56,148 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/25833819309741fd941d3286ac59a5c7, entries=150, sequenceid=398, filesize=12.0 K 2024-12-05T12:14:56,149 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/.tmp/C/346d404ba76944cead65cc71bc720ca5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/346d404ba76944cead65cc71bc720ca5 2024-12-05T12:14:56,152 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/346d404ba76944cead65cc71bc720ca5, entries=150, sequenceid=398, filesize=12.0 K 2024-12-05T12:14:56,153 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1509b956c8790d96ce37e7755c7bf6f2 in 1261ms, sequenceid=398, compaction requested=true 2024-12-05T12:14:56,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/3a870d26aa3441e4b5fccf8297113f4f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b95aa8a204434562a6316ad0a96ba520, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/503e31a5dac84bbd8ddc76a070666c26, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/be5ffbfdffb44545b4226ea729a00d23, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/28252c3f9cb54cf99c135e9cf6bf8e8f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b92ae98d4e5d44fd922fe3af469f66a5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a0a52150e76f4a3db06cf984a7d886bc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/130aa228e2a94a858f69313b77c73cd7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6f9a7f4247f641e0a02189b3faac0d1a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/2dbbd74ec2ec489884b495bbb5d12e33, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b8454dc2bec04d07947e49a44ffaeb41, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/88987a1a02e14fa8a7e9d1cd94027453, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/8989b2c62eb34a6f8ed8514de0119fd1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/641451da50cc461a8f7ec34e81f4ec56, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6dd3094dc1fb4906be63fbeb56a24b8c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/58c111355a6947c08a07102258f1af6b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/170740ff09b9484cb177e5490fe560a5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6b5770520ae949aebd8c43698259faaf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/38c2ad9cad21452582a174848c8aeac9, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6c96f011c3d449f78bfd7bc5687edc88, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/4ed310385b904296be50e8212c06bbf1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/9abe5443c67d4242a10eb42e8844a262, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a612d618d20b4051884c4ebd070d8c3d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/814180362b5e479597206a61978c82fd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/51c4789a8f1644cca2200b43f4ac7424, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/aeb1b7f6fc8344c49fa69a1dd03e43e9] to archive 2024-12-05T12:14:56,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:14:56,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/3a870d26aa3441e4b5fccf8297113f4f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/3a870d26aa3441e4b5fccf8297113f4f 2024-12-05T12:14:56,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b95aa8a204434562a6316ad0a96ba520 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b95aa8a204434562a6316ad0a96ba520 2024-12-05T12:14:56,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/503e31a5dac84bbd8ddc76a070666c26 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/503e31a5dac84bbd8ddc76a070666c26 2024-12-05T12:14:56,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/be5ffbfdffb44545b4226ea729a00d23 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/be5ffbfdffb44545b4226ea729a00d23 2024-12-05T12:14:56,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/28252c3f9cb54cf99c135e9cf6bf8e8f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/28252c3f9cb54cf99c135e9cf6bf8e8f 2024-12-05T12:14:56,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b92ae98d4e5d44fd922fe3af469f66a5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b92ae98d4e5d44fd922fe3af469f66a5 2024-12-05T12:14:56,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a0a52150e76f4a3db06cf984a7d886bc to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a0a52150e76f4a3db06cf984a7d886bc 2024-12-05T12:14:56,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/130aa228e2a94a858f69313b77c73cd7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/130aa228e2a94a858f69313b77c73cd7 2024-12-05T12:14:56,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6f9a7f4247f641e0a02189b3faac0d1a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6f9a7f4247f641e0a02189b3faac0d1a 2024-12-05T12:14:56,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/2dbbd74ec2ec489884b495bbb5d12e33 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/2dbbd74ec2ec489884b495bbb5d12e33 2024-12-05T12:14:56,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b8454dc2bec04d07947e49a44ffaeb41 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/b8454dc2bec04d07947e49a44ffaeb41 2024-12-05T12:14:56,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/88987a1a02e14fa8a7e9d1cd94027453 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/88987a1a02e14fa8a7e9d1cd94027453 2024-12-05T12:14:56,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/8989b2c62eb34a6f8ed8514de0119fd1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/8989b2c62eb34a6f8ed8514de0119fd1 2024-12-05T12:14:56,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/641451da50cc461a8f7ec34e81f4ec56 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/641451da50cc461a8f7ec34e81f4ec56 2024-12-05T12:14:56,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6dd3094dc1fb4906be63fbeb56a24b8c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6dd3094dc1fb4906be63fbeb56a24b8c 2024-12-05T12:14:56,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/58c111355a6947c08a07102258f1af6b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/58c111355a6947c08a07102258f1af6b 2024-12-05T12:14:56,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/170740ff09b9484cb177e5490fe560a5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/170740ff09b9484cb177e5490fe560a5 2024-12-05T12:14:56,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6b5770520ae949aebd8c43698259faaf to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6b5770520ae949aebd8c43698259faaf 2024-12-05T12:14:56,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/38c2ad9cad21452582a174848c8aeac9 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/38c2ad9cad21452582a174848c8aeac9 2024-12-05T12:14:56,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6c96f011c3d449f78bfd7bc5687edc88 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/6c96f011c3d449f78bfd7bc5687edc88 2024-12-05T12:14:56,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/4ed310385b904296be50e8212c06bbf1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/4ed310385b904296be50e8212c06bbf1 2024-12-05T12:14:56,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/9abe5443c67d4242a10eb42e8844a262 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/9abe5443c67d4242a10eb42e8844a262 2024-12-05T12:14:56,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a612d618d20b4051884c4ebd070d8c3d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/a612d618d20b4051884c4ebd070d8c3d 2024-12-05T12:14:56,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/814180362b5e479597206a61978c82fd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/814180362b5e479597206a61978c82fd 2024-12-05T12:14:56,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/51c4789a8f1644cca2200b43f4ac7424 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/51c4789a8f1644cca2200b43f4ac7424 2024-12-05T12:14:56,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/aeb1b7f6fc8344c49fa69a1dd03e43e9 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/aeb1b7f6fc8344c49fa69a1dd03e43e9 2024-12-05T12:14:56,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/dc7843a9ea8d4cf6be581191fb8da9b8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cb5195087fd747a5a5769dfc6be59215, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/afe2245c2ac948c3a2f70c1a3c925a2b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/6ebd5d47939245a58eafa25f37e41bd4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77b6abcc2a144140b593b58b47247a5e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/a6efd3ef77744668ad029a35a43081f9, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/baf2c64352dd44578e019505b33ee5ae, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/b82b48927d0f41dab174718ae8b610ae, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cd16608eeaf04c82ae6d7fce988eb36e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/fdc349a836c24970ae56864399c7a907, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9fb47e9865cb47eba82528216c2b96a6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9f35bf25f3e448339e1a472fda58559a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d04f37ffd0ff44c7b0d23999a397c7fd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/134b6087328e46dfa2cb8d1bc394e6cc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/593944b851374b24ae678a73ddb07ddf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77bca7672c8b404ca9b13eb86e2f5481, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/43be3ee3b4c543f8a47eeae473ae00b6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/229364dfdd2545339d6f4e5abec9ab73, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/835dd78f5ea94b7abdc2aa3d0c786db7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/1379752a72a34e408884dc05aa7d0cf6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/3b9d05ed9cde473782ec2b453bb48e6b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/e6520512ab5b4892b674d82e540055b5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/98f3ffafc77040c4a517c0e1f0cc06b4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/426b2de8c70a4dba94aa0dc704161cc0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d6ee062341d14d31964a91252898b778, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/634db6d6f29e43ee811db4a9da78eda9] to archive 2024-12-05T12:14:56,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:14:56,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/dc7843a9ea8d4cf6be581191fb8da9b8 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/dc7843a9ea8d4cf6be581191fb8da9b8 2024-12-05T12:14:56,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cb5195087fd747a5a5769dfc6be59215 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cb5195087fd747a5a5769dfc6be59215 2024-12-05T12:14:56,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/afe2245c2ac948c3a2f70c1a3c925a2b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/afe2245c2ac948c3a2f70c1a3c925a2b 2024-12-05T12:14:56,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/6ebd5d47939245a58eafa25f37e41bd4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/6ebd5d47939245a58eafa25f37e41bd4 2024-12-05T12:14:56,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77b6abcc2a144140b593b58b47247a5e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77b6abcc2a144140b593b58b47247a5e 2024-12-05T12:14:56,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/a6efd3ef77744668ad029a35a43081f9 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/a6efd3ef77744668ad029a35a43081f9 2024-12-05T12:14:56,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/baf2c64352dd44578e019505b33ee5ae to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/baf2c64352dd44578e019505b33ee5ae 2024-12-05T12:14:56,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/b82b48927d0f41dab174718ae8b610ae to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/b82b48927d0f41dab174718ae8b610ae 2024-12-05T12:14:56,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cd16608eeaf04c82ae6d7fce988eb36e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/cd16608eeaf04c82ae6d7fce988eb36e 2024-12-05T12:14:56,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/fdc349a836c24970ae56864399c7a907 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/fdc349a836c24970ae56864399c7a907 2024-12-05T12:14:56,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9fb47e9865cb47eba82528216c2b96a6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9fb47e9865cb47eba82528216c2b96a6 2024-12-05T12:14:56,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9f35bf25f3e448339e1a472fda58559a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/9f35bf25f3e448339e1a472fda58559a 2024-12-05T12:14:56,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d04f37ffd0ff44c7b0d23999a397c7fd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d04f37ffd0ff44c7b0d23999a397c7fd 2024-12-05T12:14:56,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/134b6087328e46dfa2cb8d1bc394e6cc to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/134b6087328e46dfa2cb8d1bc394e6cc 2024-12-05T12:14:56,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/593944b851374b24ae678a73ddb07ddf to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/593944b851374b24ae678a73ddb07ddf 2024-12-05T12:14:56,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77bca7672c8b404ca9b13eb86e2f5481 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/77bca7672c8b404ca9b13eb86e2f5481 2024-12-05T12:14:56,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/43be3ee3b4c543f8a47eeae473ae00b6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/43be3ee3b4c543f8a47eeae473ae00b6 2024-12-05T12:14:56,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/229364dfdd2545339d6f4e5abec9ab73 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/229364dfdd2545339d6f4e5abec9ab73 2024-12-05T12:14:56,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/835dd78f5ea94b7abdc2aa3d0c786db7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/835dd78f5ea94b7abdc2aa3d0c786db7 2024-12-05T12:14:56,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/1379752a72a34e408884dc05aa7d0cf6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/1379752a72a34e408884dc05aa7d0cf6 2024-12-05T12:14:56,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/3b9d05ed9cde473782ec2b453bb48e6b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/3b9d05ed9cde473782ec2b453bb48e6b 2024-12-05T12:14:56,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/e6520512ab5b4892b674d82e540055b5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/e6520512ab5b4892b674d82e540055b5 2024-12-05T12:14:56,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/98f3ffafc77040c4a517c0e1f0cc06b4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/98f3ffafc77040c4a517c0e1f0cc06b4 2024-12-05T12:14:56,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/426b2de8c70a4dba94aa0dc704161cc0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/426b2de8c70a4dba94aa0dc704161cc0 2024-12-05T12:14:56,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d6ee062341d14d31964a91252898b778 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/d6ee062341d14d31964a91252898b778 2024-12-05T12:14:56,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/634db6d6f29e43ee811db4a9da78eda9 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/634db6d6f29e43ee811db4a9da78eda9 2024-12-05T12:14:56,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/755cdecae8734a05876e2c837a395cef, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/3de7947ec8904206ae58b052236755c6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/d14fdfab8fd74efb917fe458d54c3700, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/f05fc69b7307451ca5c5f963dc465824, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a350d7f6dab8466491b726253c43c756, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1d7d93d9aa3a4705826ad34ab744bc39, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/42008020b0674c01af2867b751a0fa75, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/741fa51380fc4a1898aea9ac930cb704, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a5b4c314b28d4309a8480b57f4d90e6a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/03d4c3a4562944dbab11063e3e69fd34, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9a2a0598979c46c886d866a5d61e0b50, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/64a358fd323a40aa8411164b92b75635, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/6a7a9287a976427c8f6bc3b4a6eb97d8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0cb3db84d0574e238eccfc93fff2c4e3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/bbfa3465abe54c9c928ea6123e03d4a5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9276253b2e8c4dac88857cbcd76b9884, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1c1d7be279df469983406291504af839, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ddb272996a9c4a02be111304cedc1ebf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0a4c20bfc1eb4dde8cba45b18bbb5bcd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/5136d22c7327474495a947989591837b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9f01c92fcd0a4e47a7a77d2116ef31ac, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/c9a94777425241eeafaa04045341a6c4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/faf3ece209d14aa9b2cc3374a5209491, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/435132c0ed884705965c10c11ef8f2bb, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ec03839bb4064a60ae31c276cbcc7c9b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/80b424a08cca4302962708229617f26a] to archive 2024-12-05T12:14:56,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:14:56,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/755cdecae8734a05876e2c837a395cef to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/755cdecae8734a05876e2c837a395cef 2024-12-05T12:14:56,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/3de7947ec8904206ae58b052236755c6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/3de7947ec8904206ae58b052236755c6 2024-12-05T12:14:56,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/d14fdfab8fd74efb917fe458d54c3700 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/d14fdfab8fd74efb917fe458d54c3700 2024-12-05T12:14:56,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/f05fc69b7307451ca5c5f963dc465824 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/f05fc69b7307451ca5c5f963dc465824 2024-12-05T12:14:56,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a350d7f6dab8466491b726253c43c756 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a350d7f6dab8466491b726253c43c756 2024-12-05T12:14:56,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1d7d93d9aa3a4705826ad34ab744bc39 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1d7d93d9aa3a4705826ad34ab744bc39 2024-12-05T12:14:56,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/42008020b0674c01af2867b751a0fa75 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/42008020b0674c01af2867b751a0fa75 2024-12-05T12:14:56,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/741fa51380fc4a1898aea9ac930cb704 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/741fa51380fc4a1898aea9ac930cb704 2024-12-05T12:14:56,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a5b4c314b28d4309a8480b57f4d90e6a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/a5b4c314b28d4309a8480b57f4d90e6a 2024-12-05T12:14:56,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/03d4c3a4562944dbab11063e3e69fd34 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/03d4c3a4562944dbab11063e3e69fd34 2024-12-05T12:14:56,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9a2a0598979c46c886d866a5d61e0b50 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9a2a0598979c46c886d866a5d61e0b50 2024-12-05T12:14:56,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/64a358fd323a40aa8411164b92b75635 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/64a358fd323a40aa8411164b92b75635 2024-12-05T12:14:56,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/6a7a9287a976427c8f6bc3b4a6eb97d8 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/6a7a9287a976427c8f6bc3b4a6eb97d8 2024-12-05T12:14:56,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0cb3db84d0574e238eccfc93fff2c4e3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0cb3db84d0574e238eccfc93fff2c4e3 2024-12-05T12:14:56,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/bbfa3465abe54c9c928ea6123e03d4a5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/bbfa3465abe54c9c928ea6123e03d4a5 2024-12-05T12:14:56,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9276253b2e8c4dac88857cbcd76b9884 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9276253b2e8c4dac88857cbcd76b9884 2024-12-05T12:14:56,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1c1d7be279df469983406291504af839 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/1c1d7be279df469983406291504af839 2024-12-05T12:14:56,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ddb272996a9c4a02be111304cedc1ebf to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ddb272996a9c4a02be111304cedc1ebf 2024-12-05T12:14:56,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0a4c20bfc1eb4dde8cba45b18bbb5bcd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/0a4c20bfc1eb4dde8cba45b18bbb5bcd 2024-12-05T12:14:56,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/5136d22c7327474495a947989591837b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/5136d22c7327474495a947989591837b 2024-12-05T12:14:56,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9f01c92fcd0a4e47a7a77d2116ef31ac to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/9f01c92fcd0a4e47a7a77d2116ef31ac 2024-12-05T12:14:56,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/c9a94777425241eeafaa04045341a6c4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/c9a94777425241eeafaa04045341a6c4 2024-12-05T12:14:56,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/faf3ece209d14aa9b2cc3374a5209491 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/faf3ece209d14aa9b2cc3374a5209491 2024-12-05T12:14:56,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/435132c0ed884705965c10c11ef8f2bb to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/435132c0ed884705965c10c11ef8f2bb 2024-12-05T12:14:56,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ec03839bb4064a60ae31c276cbcc7c9b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/ec03839bb4064a60ae31c276cbcc7c9b 2024-12-05T12:14:56,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/80b424a08cca4302962708229617f26a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/80b424a08cca4302962708229617f26a 2024-12-05T12:14:56,229 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/recovered.edits/401.seqid, newMaxSeqId=401, maxSeqId=1 2024-12-05T12:14:56,229 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. 2024-12-05T12:14:56,229 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 1509b956c8790d96ce37e7755c7bf6f2: Waiting for close lock at 1733400894892Running coprocessor pre-close hooks at 1733400894892Disabling compacts and flushes for region at 1733400894892Disabling writes for close at 1733400894892Obtaining lock to block concurrent updates at 1733400894892Preparing flush snapshotting stores in 1509b956c8790d96ce37e7755c7bf6f2 at 1733400894892Finished memstore snapshotting TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733400894892Flushing stores of TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2. at 1733400894893 (+1 ms)Flushing 1509b956c8790d96ce37e7755c7bf6f2/A: creating writer at 1733400894893Flushing 1509b956c8790d96ce37e7755c7bf6f2/A: appending metadata at 1733400894896 (+3 ms)Flushing 1509b956c8790d96ce37e7755c7bf6f2/A: closing flushed file at 1733400894896Flushing 1509b956c8790d96ce37e7755c7bf6f2/B: creating writer at 1733400895309 (+413 ms)Flushing 1509b956c8790d96ce37e7755c7bf6f2/B: appending metadata at 1733400895312 (+3 ms)Flushing 1509b956c8790d96ce37e7755c7bf6f2/B: closing flushed file at 1733400895312Flushing 1509b956c8790d96ce37e7755c7bf6f2/C: creating writer at 1733400895724 (+412 ms)Flushing 1509b956c8790d96ce37e7755c7bf6f2/C: appending metadata at 1733400895728 (+4 ms)Flushing 1509b956c8790d96ce37e7755c7bf6f2/C: closing flushed file at 1733400895728Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9306ec4: reopening flushed file at 1733400896139 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c0ca3db: reopening flushed file at 1733400896144 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d2c243e: reopening flushed file at 1733400896149 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1509b956c8790d96ce37e7755c7bf6f2 in 1261ms, sequenceid=398, compaction requested=true at 1733400896153 (+4 ms)Writing region close event to WAL at 1733400896225 (+72 ms)Running coprocessor post-close hooks at 1733400896229 (+4 ms)Closed at 1733400896229 2024-12-05T12:14:56,231 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:56,231 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=1509b956c8790d96ce37e7755c7bf6f2, regionState=CLOSED 2024-12-05T12:14:56,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:14:56,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-05T12:14:56,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 1509b956c8790d96ce37e7755c7bf6f2, server=9ea9abad0e9b,44759,1733400796278 in 1.4940 sec 2024-12-05T12:14:56,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-05T12:14:56,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1509b956c8790d96ce37e7755c7bf6f2, UNASSIGN in 1.5000 sec 2024-12-05T12:14:56,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-05T12:14:56,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5040 sec 2024-12-05T12:14:56,239 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400896239"}]},"ts":"1733400896239"} 2024-12-05T12:14:56,240 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T12:14:56,240 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T12:14:56,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5140 sec 2024-12-05T12:14:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T12:14:56,873 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:56,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T12:14:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:56,876 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-05T12:14:56,877 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:56,881 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:56,885 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/recovered.edits] 2024-12-05T12:14:56,889 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/02f3c312d04f4bfc91abee8d394a90ae to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/02f3c312d04f4bfc91abee8d394a90ae 2024-12-05T12:14:56,891 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/066d53bd0cd74e2f87412fd4ae64d4e2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/066d53bd0cd74e2f87412fd4ae64d4e2 2024-12-05T12:14:56,893 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/c61af2580f314407aec497ba94fa5b8d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/A/c61af2580f314407aec497ba94fa5b8d 2024-12-05T12:14:56,896 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/25833819309741fd941d3286ac59a5c7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/25833819309741fd941d3286ac59a5c7 2024-12-05T12:14:56,897 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/2c874db717524ef1bc4263f5de6edfcd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/2c874db717524ef1bc4263f5de6edfcd 2024-12-05T12:14:56,898 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/31717e0fc0904fac9fd8a5a7d174ba8c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/B/31717e0fc0904fac9fd8a5a7d174ba8c 2024-12-05T12:14:56,900 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/2688699ce0c64b979d02a4c2a54f396b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/2688699ce0c64b979d02a4c2a54f396b 2024-12-05T12:14:56,902 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/346d404ba76944cead65cc71bc720ca5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/346d404ba76944cead65cc71bc720ca5 2024-12-05T12:14:56,903 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/f1243f78e96b417faeeaabb3e8f792e2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/C/f1243f78e96b417faeeaabb3e8f792e2 2024-12-05T12:14:56,906 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/recovered.edits/401.seqid to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2/recovered.edits/401.seqid 2024-12-05T12:14:56,907 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/1509b956c8790d96ce37e7755c7bf6f2 2024-12-05T12:14:56,907 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T12:14:56,910 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:56,914 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T12:14:56,916 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T12:14:56,918 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:56,918 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T12:14:56,918 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733400896918"}]},"ts":"9223372036854775807"} 2024-12-05T12:14:56,921 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T12:14:56,921 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1509b956c8790d96ce37e7755c7bf6f2, NAME => 'TestAcidGuarantees,,1733400864695.1509b956c8790d96ce37e7755c7bf6f2.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T12:14:56,921 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T12:14:56,921 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733400896921"}]},"ts":"9223372036854775807"} 2024-12-05T12:14:56,924 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T12:14:56,925 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:56,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-12-05T12:14:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-05T12:14:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T12:14:56,983 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:56,994 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 252), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=638 (was 705), ProcessCount=11 (was 11), AvailableMemoryMB=3610 (was 4475) 2024-12-05T12:14:57,004 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=638, ProcessCount=11, AvailableMemoryMB=3610 2024-12-05T12:14:57,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T12:14:57,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:14:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:57,008 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T12:14:57,009 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:57,010 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T12:14:57,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-12-05T12:14:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T12:14:57,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742163_1339 (size=1061) 2024-12-05T12:14:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T12:14:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T12:14:57,418 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e 2024-12-05T12:14:57,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742164_1340 (size=53) 2024-12-05T12:14:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T12:14:57,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:14:57,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8ac3794477a85e9bc205a48eed1522fa, disabling compactions & flushes 2024-12-05T12:14:57,826 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:57,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:57,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. after waiting 0 ms 2024-12-05T12:14:57,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:57,826 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:57,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8ac3794477a85e9bc205a48eed1522fa: Waiting for close lock at 1733400897826Disabling compacts and flushes for region at 1733400897826Disabling writes for close at 1733400897826Writing region close event to WAL at 1733400897826Closed at 1733400897826 2024-12-05T12:14:57,828 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T12:14:57,828 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733400897828"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733400897828"}]},"ts":"1733400897828"} 2024-12-05T12:14:57,831 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T12:14:57,832 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T12:14:57,832 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400897832"}]},"ts":"1733400897832"} 2024-12-05T12:14:57,834 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T12:14:57,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, ASSIGN}] 2024-12-05T12:14:57,835 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, ASSIGN 2024-12-05T12:14:57,836 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, ASSIGN; state=OFFLINE, location=9ea9abad0e9b,44759,1733400796278; forceNewPlan=false, retain=false 2024-12-05T12:14:57,987 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=8ac3794477a85e9bc205a48eed1522fa, regionState=OPENING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:57,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, ASSIGN because future has completed 2024-12-05T12:14:57,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:14:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T12:14:58,149 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:58,150 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:14:58,150 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,150 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:14:58,151 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,151 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,153 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,155 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:14:58,155 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ac3794477a85e9bc205a48eed1522fa columnFamilyName A 2024-12-05T12:14:58,155 DEBUG [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:58,156 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(327): Store=8ac3794477a85e9bc205a48eed1522fa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:14:58,156 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,157 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:14:58,157 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ac3794477a85e9bc205a48eed1522fa columnFamilyName B 2024-12-05T12:14:58,157 DEBUG [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:58,157 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(327): Store=8ac3794477a85e9bc205a48eed1522fa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:14:58,157 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,158 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:14:58,158 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ac3794477a85e9bc205a48eed1522fa columnFamilyName C 2024-12-05T12:14:58,158 DEBUG [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:14:58,159 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(327): Store=8ac3794477a85e9bc205a48eed1522fa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:14:58,159 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:58,159 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,160 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,160 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,161 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,161 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,162 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:14:58,163 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,165 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:14:58,165 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 8ac3794477a85e9bc205a48eed1522fa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69160949, jitterRate=0.030578449368476868}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:14:58,165 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:58,166 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 8ac3794477a85e9bc205a48eed1522fa: Running coprocessor pre-open hook at 1733400898151Writing region info on filesystem at 1733400898151Initializing all the Stores at 1733400898152 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400898152Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400898153 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400898153Cleaning up temporary data from old regions at 1733400898161 (+8 ms)Running coprocessor post-open hooks at 1733400898165 (+4 ms)Region opened successfully at 1733400898166 (+1 ms) 2024-12-05T12:14:58,167 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., pid=89, masterSystemTime=1733400898144 2024-12-05T12:14:58,169 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:58,169 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:58,169 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=8ac3794477a85e9bc205a48eed1522fa, regionState=OPEN, openSeqNum=2, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:58,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:14:58,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-05T12:14:58,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 in 180 msec 2024-12-05T12:14:58,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-05T12:14:58,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, ASSIGN in 339 msec 2024-12-05T12:14:58,175 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T12:14:58,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400898175"}]},"ts":"1733400898175"} 2024-12-05T12:14:58,177 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T12:14:58,178 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T12:14:58,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-12-05T12:14:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T12:14:59,154 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:14:59,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59596ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:59,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:14:59,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:14:59,157 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:14:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:14:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:14:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bb6242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:14:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:14:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:14:59,159 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:14:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32147737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:14:59,160 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:14:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:14:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:14:59,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:14:59,163 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T12:14:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ea9abad0e9b,34103,1733400795471 2024-12-05T12:14:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64d72008 2024-12-05T12:14:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T12:14:59,164 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T12:14:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T12:14:59,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:14:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T12:14:59,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742165_1341 (size=1097) 2024-12-05T12:14:59,580 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T12:14:59,580 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T12:14:59,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T12:14:59,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, REOPEN/MOVE}] 2024-12-05T12:14:59,589 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, REOPEN/MOVE 2024-12-05T12:14:59,590 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=8ac3794477a85e9bc205a48eed1522fa, regionState=CLOSING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:59,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, REOPEN/MOVE because future has completed 2024-12-05T12:14:59,591 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T12:14:59,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:14:59,744 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:59,744 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T12:14:59,744 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 8ac3794477a85e9bc205a48eed1522fa, disabling compactions & flushes 2024-12-05T12:14:59,744 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:59,744 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:59,744 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. after waiting 0 ms 2024-12-05T12:14:59,744 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:59,748 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T12:14:59,748 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:14:59,748 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 8ac3794477a85e9bc205a48eed1522fa: Waiting for close lock at 1733400899744Running coprocessor pre-close hooks at 1733400899744Disabling compacts and flushes for region at 1733400899744Disabling writes for close at 1733400899744Writing region close event to WAL at 1733400899744Running coprocessor post-close hooks at 1733400899748 (+4 ms)Closed at 1733400899748 2024-12-05T12:14:59,748 WARN [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 8ac3794477a85e9bc205a48eed1522fa to self. 2024-12-05T12:14:59,750 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:14:59,751 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=8ac3794477a85e9bc205a48eed1522fa, regionState=CLOSED 2024-12-05T12:14:59,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:14:59,754 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-05T12:14:59,754 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 in 162 msec 2024-12-05T12:14:59,755 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, REOPEN/MOVE; state=CLOSED, location=9ea9abad0e9b,44759,1733400796278; forceNewPlan=false, retain=true 2024-12-05T12:14:59,906 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=8ac3794477a85e9bc205a48eed1522fa, regionState=OPENING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:14:59,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, REOPEN/MOVE because future has completed 2024-12-05T12:14:59,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:15:00,067 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,067 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:15:00,067 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,067 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:15:00,067 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,067 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,069 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,070 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:00,070 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ac3794477a85e9bc205a48eed1522fa columnFamilyName A 2024-12-05T12:15:00,071 DEBUG [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:00,071 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(327): Store=8ac3794477a85e9bc205a48eed1522fa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:00,072 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,072 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:00,073 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ac3794477a85e9bc205a48eed1522fa columnFamilyName B 2024-12-05T12:15:00,073 DEBUG [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:00,073 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(327): Store=8ac3794477a85e9bc205a48eed1522fa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:00,073 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,074 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:00,074 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ac3794477a85e9bc205a48eed1522fa columnFamilyName C 2024-12-05T12:15:00,074 DEBUG [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:00,074 INFO [StoreOpener-8ac3794477a85e9bc205a48eed1522fa-1 {}] regionserver.HStore(327): Store=8ac3794477a85e9bc205a48eed1522fa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:00,074 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,074 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,075 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,076 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,077 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,077 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,078 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:15:00,079 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,079 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 8ac3794477a85e9bc205a48eed1522fa; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61376984, jitterRate=-0.0854116678237915}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:15:00,079 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,080 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 8ac3794477a85e9bc205a48eed1522fa: Running coprocessor pre-open hook at 1733400900067Writing region info on filesystem at 1733400900067Initializing all the Stores at 1733400900068 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400900068Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400900068Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400900068Cleaning up temporary data from old regions at 1733400900077 (+9 ms)Running coprocessor post-open hooks at 1733400900079 (+2 ms)Region opened successfully at 1733400900080 (+1 ms) 2024-12-05T12:15:00,080 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., pid=94, masterSystemTime=1733400900064 2024-12-05T12:15:00,082 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,082 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,082 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=8ac3794477a85e9bc205a48eed1522fa, regionState=OPEN, openSeqNum=5, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:15:00,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-12-05T12:15:00,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 in 175 msec 2024-12-05T12:15:00,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-05T12:15:00,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, REOPEN/MOVE in 500 msec 2024-12-05T12:15:00,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-05T12:15:00,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 505 msec 2024-12-05T12:15:00,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 925 msec 2024-12-05T12:15:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-05T12:15:00,093 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:00,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4853c07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:00,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:00,095 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69122fbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:00,096 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3152c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f7d3c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:00,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:00,098 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107725bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:00,100 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f09524, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48b614aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:00,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:00,102 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a941e73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:00,103 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9bd8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c87087, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:00,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:00,105 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61cf42b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:00,107 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f6c783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7679e723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:00,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:00,109 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:00,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:00,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:00,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b35b3f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:00,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:00,110 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db901d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f259004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:00,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:00,112 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f9e4d98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:00,114 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b6271dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526ed020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:00,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:00,116 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0afd17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:00,118 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5669674e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2ec5f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,119 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:00,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:00,120 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760b3123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:00,121 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5998b8df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ee78e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:00,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:00,123 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cebd6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:00,126 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f33f9f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ee060c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:00,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:00,130 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b753bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:00,131 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c4aad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:00,132 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:00,132 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:00,133 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:00,133 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:00,133 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:00,133 DEBUG [Thread-1570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:00,134 DEBUG [Thread-1561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:00,134 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-05T12:15:00,134 DEBUG [Thread-1561 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:00,135 DEBUG [Thread-1561 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:00,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:00,135 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:00,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:00,136 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:00,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:00,136 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:00,136 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:00,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:00,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T12:15:00,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:00,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:00,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:00,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:00,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:00,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:00,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:00,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:00,140 DEBUG [Thread-1553 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:00,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:15:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:00,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120542b8edbfda2a476c97356ae22f1c38a3_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400900151/Put/seqid=0 2024-12-05T12:15:00,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742166_1342 (size=12154) 2024-12-05T12:15:00,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400960185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400960185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400960191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400960192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400960193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T12:15:00,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T12:15:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400960302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400960302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400960303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400960303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400960303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T12:15:00,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:00,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T12:15:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400960513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400960513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400960513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400960513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400960513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,583 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:00,586 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120542b8edbfda2a476c97356ae22f1c38a3_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120542b8edbfda2a476c97356ae22f1c38a3_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:00,587 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/72c3cd5245a8405290c2efd4a6171a5c, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:00,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/72c3cd5245a8405290c2efd4a6171a5c is 175, key is test_row_0/A:col10/1733400900151/Put/seqid=0 2024-12-05T12:15:00,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742167_1343 (size=30955) 2024-12-05T12:15:00,593 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/72c3cd5245a8405290c2efd4a6171a5c 2024-12-05T12:15:00,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T12:15:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/94841b45ec3443a78463550e37ae52bf is 50, key is test_row_0/B:col10/1733400900151/Put/seqid=0 2024-12-05T12:15:00,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742168_1344 (size=12001) 2024-12-05T12:15:00,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T12:15:00,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:00,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T12:15:00,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400960823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400960823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400960823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400960823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400960823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:00,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T12:15:00,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:00,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:00,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:01,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/94841b45ec3443a78463550e37ae52bf 2024-12-05T12:15:01,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/d6437dd916224eb8872a7965cab1ed24 is 50, key is test_row_0/C:col10/1733400900151/Put/seqid=0 2024-12-05T12:15:01,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742169_1345 (size=12001) 2024-12-05T12:15:01,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T12:15:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:01,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:01,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T12:15:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:01,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T12:15:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400961334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400961334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:01,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400961334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:01,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400961334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:01,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400961334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:01,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T12:15:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:01,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:01,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/d6437dd916224eb8872a7965cab1ed24 2024-12-05T12:15:01,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/72c3cd5245a8405290c2efd4a6171a5c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/72c3cd5245a8405290c2efd4a6171a5c 2024-12-05T12:15:01,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/72c3cd5245a8405290c2efd4a6171a5c, entries=150, sequenceid=16, filesize=30.2 K 2024-12-05T12:15:01,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/94841b45ec3443a78463550e37ae52bf as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/94841b45ec3443a78463550e37ae52bf 2024-12-05T12:15:01,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/94841b45ec3443a78463550e37ae52bf, entries=150, sequenceid=16, filesize=11.7 K 2024-12-05T12:15:01,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/d6437dd916224eb8872a7965cab1ed24 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d6437dd916224eb8872a7965cab1ed24 2024-12-05T12:15:01,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d6437dd916224eb8872a7965cab1ed24, entries=150, sequenceid=16, filesize=11.7 K 2024-12-05T12:15:01,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8ac3794477a85e9bc205a48eed1522fa in 1318ms, sequenceid=16, compaction requested=false 2024-12-05T12:15:01,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:01,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T12:15:01,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:01,516 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T12:15:01,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:01,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:01,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:01,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:01,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:01,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:01,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120521abf84cd4064d90b066bf7c0214ce7a_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400900191/Put/seqid=0 2024-12-05T12:15:01,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742170_1346 (size=12154) 2024-12-05T12:15:01,767 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T12:15:01,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:01,932 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120521abf84cd4064d90b066bf7c0214ce7a_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120521abf84cd4064d90b066bf7c0214ce7a_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/c8575b3a1078407699594fbb1b3def94, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:01,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/c8575b3a1078407699594fbb1b3def94 is 175, key is test_row_0/A:col10/1733400900191/Put/seqid=0 2024-12-05T12:15:01,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742171_1347 (size=30955) 2024-12-05T12:15:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T12:15:02,340 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/c8575b3a1078407699594fbb1b3def94 2024-12-05T12:15:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:02,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:02,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/c98c37b0fe33411a94b5d0c43c86bddc is 50, key is test_row_0/B:col10/1733400900191/Put/seqid=0 2024-12-05T12:15:02,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742172_1348 (size=12001) 2024-12-05T12:15:02,366 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/c98c37b0fe33411a94b5d0c43c86bddc 2024-12-05T12:15:02,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400962359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400962359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400962365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400962368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400962369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/ebe5c2fc71394a288f4195dd0be9e7e7 is 50, key is test_row_0/C:col10/1733400900191/Put/seqid=0 2024-12-05T12:15:02,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742173_1349 (size=12001) 2024-12-05T12:15:02,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400962473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400962473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400962483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400962483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400962483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400962682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400962682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400962693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400962693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:02,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400962693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:02,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:02,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:02,784 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/ebe5c2fc71394a288f4195dd0be9e7e7 2024-12-05T12:15:02,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/c8575b3a1078407699594fbb1b3def94 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/c8575b3a1078407699594fbb1b3def94 2024-12-05T12:15:02,794 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/c8575b3a1078407699594fbb1b3def94, entries=150, sequenceid=36, filesize=30.2 K 2024-12-05T12:15:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/c98c37b0fe33411a94b5d0c43c86bddc as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/c98c37b0fe33411a94b5d0c43c86bddc 2024-12-05T12:15:02,809 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/c98c37b0fe33411a94b5d0c43c86bddc, entries=150, sequenceid=36, filesize=11.7 K 2024-12-05T12:15:02,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/ebe5c2fc71394a288f4195dd0be9e7e7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/ebe5c2fc71394a288f4195dd0be9e7e7 2024-12-05T12:15:02,815 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/ebe5c2fc71394a288f4195dd0be9e7e7, entries=150, sequenceid=36, filesize=11.7 K 2024-12-05T12:15:02,816 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8ac3794477a85e9bc205a48eed1522fa in 1300ms, sequenceid=36, compaction requested=false 2024-12-05T12:15:02,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:02,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:02,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-05T12:15:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-05T12:15:02,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-05T12:15:02,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6810 sec 2024-12-05T12:15:02,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.6880 sec 2024-12-05T12:15:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:02,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T12:15:02,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:02,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:02,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:02,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:02,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:02,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:03,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205641cf6a3355d4415b6fe039f093d90c8_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400902364/Put/seqid=0 2024-12-05T12:15:03,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742174_1350 (size=14594) 2024-12-05T12:15:03,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400963023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400963024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400963026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400963027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400963028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400963142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400963142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400963142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400963143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400963143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400963352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400963352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400963353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400963353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400963353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,408 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:03,411 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205641cf6a3355d4415b6fe039f093d90c8_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205641cf6a3355d4415b6fe039f093d90c8_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:03,412 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/5a92e7c41abe435ea12b35f495a21936, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:03,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/5a92e7c41abe435ea12b35f495a21936 is 175, key is test_row_0/A:col10/1733400902364/Put/seqid=0 2024-12-05T12:15:03,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742175_1351 (size=39549) 2024-12-05T12:15:03,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400963662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400963662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400963663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400963663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400963663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:03,817 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/5a92e7c41abe435ea12b35f495a21936 2024-12-05T12:15:03,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/286002a02e09455eb76f6c6be23a9260 is 50, key is test_row_0/B:col10/1733400902364/Put/seqid=0 2024-12-05T12:15:03,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742176_1352 (size=12001) 2024-12-05T12:15:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400964173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400964173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:04,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400964173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:04,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400964183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:04,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400964183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:04,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/286002a02e09455eb76f6c6be23a9260 2024-12-05T12:15:04,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/207a697343314feba50911417627662a is 50, key is test_row_0/C:col10/1733400902364/Put/seqid=0 2024-12-05T12:15:04,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742177_1353 (size=12001) 2024-12-05T12:15:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T12:15:04,293 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:04,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-05T12:15:04,295 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T12:15:04,296 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:04,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T12:15:04,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T12:15:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:04,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:04,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T12:15:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:04,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T12:15:04,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/207a697343314feba50911417627662a 2024-12-05T12:15:04,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/5a92e7c41abe435ea12b35f495a21936 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/5a92e7c41abe435ea12b35f495a21936 2024-12-05T12:15:04,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/5a92e7c41abe435ea12b35f495a21936, entries=200, sequenceid=49, filesize=38.6 K 2024-12-05T12:15:04,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/286002a02e09455eb76f6c6be23a9260 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/286002a02e09455eb76f6c6be23a9260 2024-12-05T12:15:04,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/286002a02e09455eb76f6c6be23a9260, entries=150, sequenceid=49, filesize=11.7 K 2024-12-05T12:15:04,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/207a697343314feba50911417627662a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/207a697343314feba50911417627662a 2024-12-05T12:15:04,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/207a697343314feba50911417627662a, entries=150, sequenceid=49, filesize=11.7 K 2024-12-05T12:15:04,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8ac3794477a85e9bc205a48eed1522fa in 1660ms, sequenceid=49, compaction requested=true 2024-12-05T12:15:04,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:04,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:04,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:04,657 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:04,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:04,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:04,657 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:04,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:04,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:04,657 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:04,658 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:04,658 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/A is initiating minor compaction (all files) 2024-12-05T12:15:04,658 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/B is initiating minor compaction (all files) 2024-12-05T12:15:04,658 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/B in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:04,658 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/A in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:04,658 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/94841b45ec3443a78463550e37ae52bf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/c98c37b0fe33411a94b5d0c43c86bddc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/286002a02e09455eb76f6c6be23a9260] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=35.2 K 2024-12-05T12:15:04,658 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/72c3cd5245a8405290c2efd4a6171a5c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/c8575b3a1078407699594fbb1b3def94, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/5a92e7c41abe435ea12b35f495a21936] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=99.1 K 2024-12-05T12:15:04,658 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:04,658 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/72c3cd5245a8405290c2efd4a6171a5c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/c8575b3a1078407699594fbb1b3def94, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/5a92e7c41abe435ea12b35f495a21936] 2024-12-05T12:15:04,658 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 94841b45ec3443a78463550e37ae52bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733400900145 2024-12-05T12:15:04,658 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72c3cd5245a8405290c2efd4a6171a5c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733400900145 2024-12-05T12:15:04,658 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting c98c37b0fe33411a94b5d0c43c86bddc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733400900183 2024-12-05T12:15:04,658 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8575b3a1078407699594fbb1b3def94, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733400900183 2024-12-05T12:15:04,658 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 286002a02e09455eb76f6c6be23a9260, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733400902364 2024-12-05T12:15:04,659 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a92e7c41abe435ea12b35f495a21936, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733400902358 2024-12-05T12:15:04,664 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:04,666 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:04,666 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205f2232b1038e34863a115b32c4eb69e89_8ac3794477a85e9bc205a48eed1522fa store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:04,666 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/ee087b1aaa7c413a898faace499ff00a is 50, key is test_row_0/B:col10/1733400902364/Put/seqid=0 2024-12-05T12:15:04,668 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205f2232b1038e34863a115b32c4eb69e89_8ac3794477a85e9bc205a48eed1522fa, store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:04,668 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f2232b1038e34863a115b32c4eb69e89_8ac3794477a85e9bc205a48eed1522fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:04,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742179_1355 (size=4469) 2024-12-05T12:15:04,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742178_1354 (size=12104) 2024-12-05T12:15:04,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T12:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:04,759 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T12:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:04,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051a8b4328103841739b976060cc68c520_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400903014/Put/seqid=0 2024-12-05T12:15:04,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742180_1356 (size=12154) 2024-12-05T12:15:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:04,792 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051a8b4328103841739b976060cc68c520_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051a8b4328103841739b976060cc68c520_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:04,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:04,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d is 175, key is test_row_0/A:col10/1733400903014/Put/seqid=0 2024-12-05T12:15:04,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742181_1357 (size=30955) 2024-12-05T12:15:04,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T12:15:05,080 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#A#compaction#301 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:05,081 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/8ae33c487d9d451895d7a26ecfc36656 is 175, key is test_row_0/A:col10/1733400902364/Put/seqid=0 2024-12-05T12:15:05,091 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/ee087b1aaa7c413a898faace499ff00a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/ee087b1aaa7c413a898faace499ff00a 2024-12-05T12:15:05,095 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/B of 8ac3794477a85e9bc205a48eed1522fa into ee087b1aaa7c413a898faace499ff00a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:05,095 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:05,095 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/B, priority=13, startTime=1733400904657; duration=0sec 2024-12-05T12:15:05,095 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:05,095 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:B 2024-12-05T12:15:05,095 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:05,097 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:05,097 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/C is initiating minor compaction (all files) 2024-12-05T12:15:05,097 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/C in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:05,097 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d6437dd916224eb8872a7965cab1ed24, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/ebe5c2fc71394a288f4195dd0be9e7e7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/207a697343314feba50911417627662a] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=35.2 K 2024-12-05T12:15:05,099 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d6437dd916224eb8872a7965cab1ed24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733400900145 2024-12-05T12:15:05,099 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting ebe5c2fc71394a288f4195dd0be9e7e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733400900183 2024-12-05T12:15:05,099 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 207a697343314feba50911417627662a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733400902364 2024-12-05T12:15:05,107 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:05,107 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/71c02ffcaf4e4cda928b8935b974c9da is 50, key is test_row_0/C:col10/1733400902364/Put/seqid=0 2024-12-05T12:15:05,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742182_1358 (size=31058) 2024-12-05T12:15:05,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742183_1359 (size=12104) 2024-12-05T12:15:05,142 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/71c02ffcaf4e4cda928b8935b974c9da as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/71c02ffcaf4e4cda928b8935b974c9da 2024-12-05T12:15:05,147 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/C of 8ac3794477a85e9bc205a48eed1522fa into 71c02ffcaf4e4cda928b8935b974c9da(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:05,147 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:05,147 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/C, priority=13, startTime=1733400904657; duration=0sec 2024-12-05T12:15:05,147 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:05,147 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:C 2024-12-05T12:15:05,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:05,200 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d 2024-12-05T12:15:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/4ec00d1b785d4e54b085258c8ecd1b36 is 50, key is test_row_0/B:col10/1733400903014/Put/seqid=0 2024-12-05T12:15:05,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400965201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400965202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400965202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400965204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,211 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 8ac3794477a85e9bc205a48eed1522fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400965209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742184_1360 (size=12001) 2024-12-05T12:15:05,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400965312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400965312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400965323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400965323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T12:15:05,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400965523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400965523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,529 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/8ae33c487d9d451895d7a26ecfc36656 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/8ae33c487d9d451895d7a26ecfc36656 2024-12-05T12:15:05,533 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/A of 8ac3794477a85e9bc205a48eed1522fa into 8ae33c487d9d451895d7a26ecfc36656(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:05,533 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:05,533 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/A, priority=13, startTime=1733400904657; duration=0sec 2024-12-05T12:15:05,534 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:05,534 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:A 2024-12-05T12:15:05,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400965533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400965533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,620 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/4ec00d1b785d4e54b085258c8ecd1b36 2024-12-05T12:15:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/bec96ee7803e4c079fb4ef8c0ff3075a is 50, key is test_row_0/C:col10/1733400903014/Put/seqid=0 2024-12-05T12:15:05,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742185_1361 (size=12001) 2024-12-05T12:15:05,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400965832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400965833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400965843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400965843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,033 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/bec96ee7803e4c079fb4ef8c0ff3075a 2024-12-05T12:15:06,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d 2024-12-05T12:15:06,040 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d, entries=150, sequenceid=68, filesize=30.2 K 2024-12-05T12:15:06,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/4ec00d1b785d4e54b085258c8ecd1b36 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/4ec00d1b785d4e54b085258c8ecd1b36 2024-12-05T12:15:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,049 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/4ec00d1b785d4e54b085258c8ecd1b36, entries=150, sequenceid=68, filesize=11.7 K 2024-12-05T12:15:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/bec96ee7803e4c079fb4ef8c0ff3075a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bec96ee7803e4c079fb4ef8c0ff3075a 2024-12-05T12:15:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,056 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bec96ee7803e4c079fb4ef8c0ff3075a, entries=150, sequenceid=68, filesize=11.7 K 2024-12-05T12:15:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,057 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8ac3794477a85e9bc205a48eed1522fa in 1298ms, sequenceid=68, compaction requested=false 2024-12-05T12:15:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-05T12:15:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-05T12:15:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-05T12:15:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7620 sec 2024-12-05T12:15:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.7680 sec 2024-12-05T12:15:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:06,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T12:15:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120583f22d8818bb4cdaac01d519ac582e21_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400906363/Put/seqid=0 2024-12-05T12:15:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742186_1362 (size=12154) 2024-12-05T12:15:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400966414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400966420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400966422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400966422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T12:15:06,443 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:06,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-05T12:15:06,446 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T12:15:06,446 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:06,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:06,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400966532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400966533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400966543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400966543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T12:15:06,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:06,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:06,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:06,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:06,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:06,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:06,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:06,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:06,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:06,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:06,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:06,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:06,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400966752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400966752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T12:15:06,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400966763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:06,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400966763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:06,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:06,785 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:06,789 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120583f22d8818bb4cdaac01d519ac582e21_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120583f22d8818bb4cdaac01d519ac582e21_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:06,790 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/18d19aba699c4d82bd998fb2ade6fcf2, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:06,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/18d19aba699c4d82bd998fb2ade6fcf2 is 175, key is test_row_0/A:col10/1733400906363/Put/seqid=0 2024-12-05T12:15:06,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742187_1363 (size=30951) 2024-12-05T12:15:06,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:06,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:06,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:06,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:06,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:06,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:07,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400967062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400967063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T12:15:07,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400967073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400967073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:07,194 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/18d19aba699c4d82bd998fb2ade6fcf2 2024-12-05T12:15:07,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/0e6f3dfbaaf841f293b3c28e63a02357 is 50, key is test_row_0/B:col10/1733400906363/Put/seqid=0 2024-12-05T12:15:07,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742188_1364 (size=9657) 2024-12-05T12:15:07,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:07,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:07,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400967232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:07,236 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 8ac3794477a85e9bc205a48eed1522fa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:07,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:07,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:07,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400967573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:07,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400967574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T12:15:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400967583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400967583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:07,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:07,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/0e6f3dfbaaf841f293b3c28e63a02357 2024-12-05T12:15:07,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/d29995ca9a0b4161b62dcd6047338eef is 50, key is test_row_0/C:col10/1733400906363/Put/seqid=0 2024-12-05T12:15:07,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742189_1365 (size=9657) 2024-12-05T12:15:07,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:07,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:07,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:07,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:07,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:08,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/d29995ca9a0b4161b62dcd6047338eef 2024-12-05T12:15:08,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/18d19aba699c4d82bd998fb2ade6fcf2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/18d19aba699c4d82bd998fb2ade6fcf2 2024-12-05T12:15:08,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/18d19aba699c4d82bd998fb2ade6fcf2, entries=150, sequenceid=85, filesize=30.2 K 2024-12-05T12:15:08,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/0e6f3dfbaaf841f293b3c28e63a02357 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/0e6f3dfbaaf841f293b3c28e63a02357 2024-12-05T12:15:08,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/0e6f3dfbaaf841f293b3c28e63a02357, entries=100, sequenceid=85, filesize=9.4 K 2024-12-05T12:15:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/d29995ca9a0b4161b62dcd6047338eef as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d29995ca9a0b4161b62dcd6047338eef 2024-12-05T12:15:08,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d29995ca9a0b4161b62dcd6047338eef, entries=100, sequenceid=85, filesize=9.4 K 2024-12-05T12:15:08,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 8ac3794477a85e9bc205a48eed1522fa in 1666ms, sequenceid=85, compaction requested=true 2024-12-05T12:15:08,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:08,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:08,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:08,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:08,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:08,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:08,030 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:08,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:08,030 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:08,031 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:08,031 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:08,031 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/A is initiating minor compaction (all files) 2024-12-05T12:15:08,031 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/B is initiating minor compaction (all files) 2024-12-05T12:15:08,031 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/A in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:08,031 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/B in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:08,031 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/ee087b1aaa7c413a898faace499ff00a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/4ec00d1b785d4e54b085258c8ecd1b36, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/0e6f3dfbaaf841f293b3c28e63a02357] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=33.0 K 2024-12-05T12:15:08,031 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/8ae33c487d9d451895d7a26ecfc36656, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/18d19aba699c4d82bd998fb2ade6fcf2] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=90.8 K 2024-12-05T12:15:08,031 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:08,031 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/8ae33c487d9d451895d7a26ecfc36656, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/18d19aba699c4d82bd998fb2ade6fcf2] 2024-12-05T12:15:08,031 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting ee087b1aaa7c413a898faace499ff00a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733400902364 2024-12-05T12:15:08,032 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ec00d1b785d4e54b085258c8ecd1b36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733400903014 2024-12-05T12:15:08,032 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ae33c487d9d451895d7a26ecfc36656, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733400902364 2024-12-05T12:15:08,032 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e6f3dfbaaf841f293b3c28e63a02357, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733400905194 2024-12-05T12:15:08,032 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddcc48ca4c8a40a59f3fa6a8f33ea10d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733400903014 2024-12-05T12:15:08,032 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18d19aba699c4d82bd998fb2ade6fcf2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733400905194 2024-12-05T12:15:08,037 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#B#compaction#309 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:08,037 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/8332d4d5a5454d34a14bb95f8c7bc381 is 50, key is test_row_0/B:col10/1733400906363/Put/seqid=0 2024-12-05T12:15:08,038 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:08,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742190_1366 (size=12207) 2024-12-05T12:15:08,047 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205114f433fc17740b98f4bb747b4259dae_8ac3794477a85e9bc205a48eed1522fa store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:08,048 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205114f433fc17740b98f4bb747b4259dae_8ac3794477a85e9bc205a48eed1522fa, store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:08,049 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205114f433fc17740b98f4bb747b4259dae_8ac3794477a85e9bc205a48eed1522fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:08,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742191_1367 (size=4469) 2024-12-05T12:15:08,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T12:15:08,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:08,125 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T12:15:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:08,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cbc0bc574acd4f85baa31bc785c10ccb_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400906420/Put/seqid=0 2024-12-05T12:15:08,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742192_1368 (size=12154) 2024-12-05T12:15:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:08,155 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cbc0bc574acd4f85baa31bc785c10ccb_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cbc0bc574acd4f85baa31bc785c10ccb_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:08,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/3d1b15dc0ccb44fca7ff779002debb5b, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:08,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/3d1b15dc0ccb44fca7ff779002debb5b is 175, key is test_row_0/A:col10/1733400906420/Put/seqid=0 2024-12-05T12:15:08,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742193_1369 (size=30955) 2024-12-05T12:15:08,167 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/3d1b15dc0ccb44fca7ff779002debb5b 2024-12-05T12:15:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/421d6d2f88664886866bd69fa13d3db5 is 50, key is test_row_0/B:col10/1733400906420/Put/seqid=0 2024-12-05T12:15:08,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742194_1370 (size=12001) 2024-12-05T12:15:08,185 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/421d6d2f88664886866bd69fa13d3db5 2024-12-05T12:15:08,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/0454fdad311d4e8ab27a2871e48d8ed7 is 50, key is test_row_0/C:col10/1733400906420/Put/seqid=0 2024-12-05T12:15:08,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742195_1371 (size=12001) 2024-12-05T12:15:08,450 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/8332d4d5a5454d34a14bb95f8c7bc381 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/8332d4d5a5454d34a14bb95f8c7bc381 2024-12-05T12:15:08,454 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/B of 8ac3794477a85e9bc205a48eed1522fa into 8332d4d5a5454d34a14bb95f8c7bc381(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:08,454 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:08,454 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/B, priority=13, startTime=1733400908030; duration=0sec 2024-12-05T12:15:08,454 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:08,454 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:B 2024-12-05T12:15:08,454 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:08,455 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:08,455 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/C is initiating minor compaction (all files) 2024-12-05T12:15:08,455 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/C in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:08,455 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/71c02ffcaf4e4cda928b8935b974c9da, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bec96ee7803e4c079fb4ef8c0ff3075a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d29995ca9a0b4161b62dcd6047338eef] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=33.0 K 2024-12-05T12:15:08,455 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 71c02ffcaf4e4cda928b8935b974c9da, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733400902364 2024-12-05T12:15:08,456 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting bec96ee7803e4c079fb4ef8c0ff3075a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733400903014 2024-12-05T12:15:08,456 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d29995ca9a0b4161b62dcd6047338eef, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733400905194 2024-12-05T12:15:08,457 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#A#compaction#310 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:08,457 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/1e0685a9aab045a8a38fa8f34ac47262 is 175, key is test_row_0/A:col10/1733400906363/Put/seqid=0 2024-12-05T12:15:08,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742196_1372 (size=31268) 2024-12-05T12:15:08,464 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:08,465 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/2f188a71e253445b9af1c86556d8de6e is 50, key is test_row_0/C:col10/1733400906363/Put/seqid=0 2024-12-05T12:15:08,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742197_1373 (size=12207) 2024-12-05T12:15:08,484 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/2f188a71e253445b9af1c86556d8de6e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/2f188a71e253445b9af1c86556d8de6e 2024-12-05T12:15:08,488 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/C of 8ac3794477a85e9bc205a48eed1522fa into 2f188a71e253445b9af1c86556d8de6e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:08,488 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:08,489 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/C, priority=13, startTime=1733400908030; duration=0sec 2024-12-05T12:15:08,489 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:08,489 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:C 2024-12-05T12:15:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T12:15:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:08,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:08,598 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/0454fdad311d4e8ab27a2871e48d8ed7 2024-12-05T12:15:08,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/3d1b15dc0ccb44fca7ff779002debb5b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/3d1b15dc0ccb44fca7ff779002debb5b 2024-12-05T12:15:08,606 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/3d1b15dc0ccb44fca7ff779002debb5b, entries=150, sequenceid=104, filesize=30.2 K 2024-12-05T12:15:08,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/421d6d2f88664886866bd69fa13d3db5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/421d6d2f88664886866bd69fa13d3db5 2024-12-05T12:15:08,610 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/421d6d2f88664886866bd69fa13d3db5, entries=150, sequenceid=104, filesize=11.7 K 2024-12-05T12:15:08,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/0454fdad311d4e8ab27a2871e48d8ed7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/0454fdad311d4e8ab27a2871e48d8ed7 2024-12-05T12:15:08,613 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/0454fdad311d4e8ab27a2871e48d8ed7, entries=150, sequenceid=104, filesize=11.7 K 2024-12-05T12:15:08,614 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8ac3794477a85e9bc205a48eed1522fa in 489ms, sequenceid=104, compaction requested=false 2024-12-05T12:15:08,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:08,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:08,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-05T12:15:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-05T12:15:08,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-05T12:15:08,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1690 sec 2024-12-05T12:15:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:08,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T12:15:08,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:08,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:08,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:08,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.1740 sec 2024-12-05T12:15:08,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:08,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:08,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:08,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205acbe386c8d3044aaa7834eee6dc4495f_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400908617/Put/seqid=0 2024-12-05T12:15:08,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742198_1374 (size=12154) 2024-12-05T12:15:08,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400968646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400968647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400968647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400968652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400968763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400968763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400968772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400968772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,864 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/1e0685a9aab045a8a38fa8f34ac47262 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/1e0685a9aab045a8a38fa8f34ac47262 2024-12-05T12:15:08,868 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/A of 8ac3794477a85e9bc205a48eed1522fa into 1e0685a9aab045a8a38fa8f34ac47262(size=30.5 K), total size for store is 60.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:08,869 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:08,869 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/A, priority=13, startTime=1733400908030; duration=0sec 2024-12-05T12:15:08,869 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:08,869 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:A 2024-12-05T12:15:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400968973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400968973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400968982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:08,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400968982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:09,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:09,036 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205acbe386c8d3044aaa7834eee6dc4495f_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205acbe386c8d3044aaa7834eee6dc4495f_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:09,037 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/9fb37f360aef460a9a2fa5bf3b8af658, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:09,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/9fb37f360aef460a9a2fa5bf3b8af658 is 175, key is test_row_0/A:col10/1733400908617/Put/seqid=0 2024-12-05T12:15:09,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742199_1375 (size=30955) 2024-12-05T12:15:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400969283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400969283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:09,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400969293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:09,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400969293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:09,443 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/9fb37f360aef460a9a2fa5bf3b8af658 2024-12-05T12:15:09,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/60ea64a005004759b8f809ad005bc8cc is 50, key is test_row_0/B:col10/1733400908617/Put/seqid=0 2024-12-05T12:15:09,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742200_1376 (size=12001) 2024-12-05T12:15:09,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400969793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:09,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400969803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:09,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400969803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:09,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400969803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:09,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/60ea64a005004759b8f809ad005bc8cc 2024-12-05T12:15:09,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/c91b4f61258444be87cf1e7a89aed3ed is 50, key is test_row_0/C:col10/1733400908617/Put/seqid=0 2024-12-05T12:15:09,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742201_1377 (size=12001) 2024-12-05T12:15:10,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/c91b4f61258444be87cf1e7a89aed3ed 2024-12-05T12:15:10,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/9fb37f360aef460a9a2fa5bf3b8af658 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/9fb37f360aef460a9a2fa5bf3b8af658 2024-12-05T12:15:10,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/9fb37f360aef460a9a2fa5bf3b8af658, entries=150, sequenceid=122, filesize=30.2 K 2024-12-05T12:15:10,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/60ea64a005004759b8f809ad005bc8cc as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/60ea64a005004759b8f809ad005bc8cc 2024-12-05T12:15:10,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/60ea64a005004759b8f809ad005bc8cc, entries=150, sequenceid=122, filesize=11.7 K 2024-12-05T12:15:10,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/c91b4f61258444be87cf1e7a89aed3ed as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/c91b4f61258444be87cf1e7a89aed3ed 2024-12-05T12:15:10,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/c91b4f61258444be87cf1e7a89aed3ed, entries=150, sequenceid=122, filesize=11.7 K 2024-12-05T12:15:10,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8ac3794477a85e9bc205a48eed1522fa in 1683ms, sequenceid=122, compaction requested=true 2024-12-05T12:15:10,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:10,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:10,300 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:10,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:10,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:10,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:10,300 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:10,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:10,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:10,301 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:10,301 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93178 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:10,301 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/B is initiating minor compaction (all files) 2024-12-05T12:15:10,301 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/A is initiating minor compaction (all files) 2024-12-05T12:15:10,301 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/B in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:10,301 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/A in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:10,301 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/8332d4d5a5454d34a14bb95f8c7bc381, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/421d6d2f88664886866bd69fa13d3db5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/60ea64a005004759b8f809ad005bc8cc] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=35.4 K 2024-12-05T12:15:10,301 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/1e0685a9aab045a8a38fa8f34ac47262, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/3d1b15dc0ccb44fca7ff779002debb5b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/9fb37f360aef460a9a2fa5bf3b8af658] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=91.0 K 2024-12-05T12:15:10,301 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:10,301 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/1e0685a9aab045a8a38fa8f34ac47262, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/3d1b15dc0ccb44fca7ff779002debb5b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/9fb37f360aef460a9a2fa5bf3b8af658] 2024-12-05T12:15:10,301 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 8332d4d5a5454d34a14bb95f8c7bc381, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733400903027 2024-12-05T12:15:10,301 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e0685a9aab045a8a38fa8f34ac47262, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733400903027 2024-12-05T12:15:10,301 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 421d6d2f88664886866bd69fa13d3db5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733400906394 2024-12-05T12:15:10,302 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d1b15dc0ccb44fca7ff779002debb5b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733400906394 2024-12-05T12:15:10,302 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ea64a005004759b8f809ad005bc8cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733400908604 2024-12-05T12:15:10,302 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fb37f360aef460a9a2fa5bf3b8af658, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733400908604 2024-12-05T12:15:10,307 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:10,308 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205262feb303cfb4b128b7518d90fc3cb7e_8ac3794477a85e9bc205a48eed1522fa store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:10,308 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#B#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:10,309 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/456ae25751b6408ca4b3fe6e7164f359 is 50, key is test_row_0/B:col10/1733400908617/Put/seqid=0 2024-12-05T12:15:10,309 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205262feb303cfb4b128b7518d90fc3cb7e_8ac3794477a85e9bc205a48eed1522fa, store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:10,310 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205262feb303cfb4b128b7518d90fc3cb7e_8ac3794477a85e9bc205a48eed1522fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:10,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742202_1378 (size=4469) 2024-12-05T12:15:10,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742203_1379 (size=12309) 2024-12-05T12:15:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T12:15:10,613 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:10,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-05T12:15:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T12:15:10,615 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:10,616 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:10,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T12:15:10,729 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:10,729 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/754bd9a5a8aa46e7ba6d92d57b0090ee is 175, key is test_row_0/A:col10/1733400908617/Put/seqid=0 2024-12-05T12:15:10,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742204_1380 (size=31263) 2024-12-05T12:15:10,746 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/456ae25751b6408ca4b3fe6e7164f359 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/456ae25751b6408ca4b3fe6e7164f359 2024-12-05T12:15:10,752 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/B of 8ac3794477a85e9bc205a48eed1522fa into 456ae25751b6408ca4b3fe6e7164f359(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:10,752 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:10,752 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/B, priority=13, startTime=1733400910300; duration=0sec 2024-12-05T12:15:10,752 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:10,752 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:B 2024-12-05T12:15:10,752 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:10,753 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:10,753 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/C is initiating minor compaction (all files) 2024-12-05T12:15:10,753 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/C in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:10,753 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/2f188a71e253445b9af1c86556d8de6e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/0454fdad311d4e8ab27a2871e48d8ed7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/c91b4f61258444be87cf1e7a89aed3ed] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=35.4 K 2024-12-05T12:15:10,755 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f188a71e253445b9af1c86556d8de6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733400903027 2024-12-05T12:15:10,755 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 0454fdad311d4e8ab27a2871e48d8ed7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733400906394 2024-12-05T12:15:10,755 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting c91b4f61258444be87cf1e7a89aed3ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733400908604 2024-12-05T12:15:10,766 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:10,766 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/92f3eea0048a42909be6c5acb95ac974 is 50, key is test_row_0/C:col10/1733400908617/Put/seqid=0 2024-12-05T12:15:10,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T12:15:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:10,768 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T12:15:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:10,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120584af62d8d6f74e61bf347f57c3b5d4db_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400908645/Put/seqid=0 2024-12-05T12:15:10,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742205_1381 (size=12309) 2024-12-05T12:15:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:10,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:10,809 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/92f3eea0048a42909be6c5acb95ac974 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/92f3eea0048a42909be6c5acb95ac974 2024-12-05T12:15:10,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742206_1382 (size=12304) 2024-12-05T12:15:10,815 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/C of 8ac3794477a85e9bc205a48eed1522fa into 92f3eea0048a42909be6c5acb95ac974(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:10,815 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:10,815 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/C, priority=13, startTime=1733400910300; duration=0sec 2024-12-05T12:15:10,815 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:10,815 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:C 2024-12-05T12:15:10,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400970839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400970845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:10,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400970847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:10,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400970849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T12:15:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400970952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400970953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:10,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:10,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400970963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400970963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,140 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/754bd9a5a8aa46e7ba6d92d57b0090ee as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/754bd9a5a8aa46e7ba6d92d57b0090ee 2024-12-05T12:15:11,144 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/A of 8ac3794477a85e9bc205a48eed1522fa into 754bd9a5a8aa46e7ba6d92d57b0090ee(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:11,144 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:11,144 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/A, priority=13, startTime=1733400910300; duration=0sec 2024-12-05T12:15:11,145 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:11,145 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:A 2024-12-05T12:15:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400971162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400971163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400971173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400971173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:11,218 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120584af62d8d6f74e61bf347f57c3b5d4db_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120584af62d8d6f74e61bf347f57c3b5d4db_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:11,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/317300866d8444419d95989f7d07e592, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:11,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/317300866d8444419d95989f7d07e592 is 175, key is test_row_0/A:col10/1733400908645/Put/seqid=0 2024-12-05T12:15:11,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742207_1383 (size=31105) 2024-12-05T12:15:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T12:15:11,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53524 deadline: 1733400971263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,266 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 8ac3794477a85e9bc205a48eed1522fa of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400971473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400971473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400971483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400971483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,623 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/317300866d8444419d95989f7d07e592 2024-12-05T12:15:11,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/43daeeeb54f246ad963de4a2f0643b07 is 50, key is test_row_0/B:col10/1733400908645/Put/seqid=0 2024-12-05T12:15:11,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742208_1384 (size=12151) 2024-12-05T12:15:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T12:15:11,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400971993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400971993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400971993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400971993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:12,033 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/43daeeeb54f246ad963de4a2f0643b07 2024-12-05T12:15:12,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/5f6e6a9ec1ad4f0384c46ff9ceb13c77 is 50, key is test_row_0/C:col10/1733400908645/Put/seqid=0 2024-12-05T12:15:12,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742209_1385 (size=12151) 2024-12-05T12:15:12,497 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/5f6e6a9ec1ad4f0384c46ff9ceb13c77 2024-12-05T12:15:12,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/317300866d8444419d95989f7d07e592 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/317300866d8444419d95989f7d07e592 2024-12-05T12:15:12,505 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/317300866d8444419d95989f7d07e592, entries=150, sequenceid=140, filesize=30.4 K 2024-12-05T12:15:12,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/43daeeeb54f246ad963de4a2f0643b07 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/43daeeeb54f246ad963de4a2f0643b07 2024-12-05T12:15:12,510 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/43daeeeb54f246ad963de4a2f0643b07, entries=150, sequenceid=140, filesize=11.9 K 2024-12-05T12:15:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/5f6e6a9ec1ad4f0384c46ff9ceb13c77 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/5f6e6a9ec1ad4f0384c46ff9ceb13c77 2024-12-05T12:15:12,515 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/5f6e6a9ec1ad4f0384c46ff9ceb13c77, entries=150, sequenceid=140, filesize=11.9 K 2024-12-05T12:15:12,516 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8ac3794477a85e9bc205a48eed1522fa in 1748ms, sequenceid=140, compaction requested=false 2024-12-05T12:15:12,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:12,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:12,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-05T12:15:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-05T12:15:12,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-05T12:15:12,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9010 sec 2024-12-05T12:15:12,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.9070 sec 2024-12-05T12:15:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T12:15:12,763 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:12,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-05T12:15:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T12:15:12,765 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:12,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:12,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T12:15:12,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T12:15:12,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:12,918 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T12:15:12,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:12,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:12,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:12,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:12,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:12,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:12,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050ed37c524e0e4650a01f784c19424797_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400910825/Put/seqid=0 2024-12-05T12:15:12,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742210_1386 (size=12304) 2024-12-05T12:15:12,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:12,947 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050ed37c524e0e4650a01f784c19424797_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050ed37c524e0e4650a01f784c19424797_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/dfbe4aafd13f4875a8546ea9b9ac9d4b, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:12,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/dfbe4aafd13f4875a8546ea9b9ac9d4b is 175, key is test_row_0/A:col10/1733400910825/Put/seqid=0 2024-12-05T12:15:12,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742211_1387 (size=31105) 2024-12-05T12:15:12,990 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/dfbe4aafd13f4875a8546ea9b9ac9d4b 2024-12-05T12:15:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:13,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:13,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/d4be887797684392ae0909ad400f4da0 is 50, key is test_row_0/B:col10/1733400910825/Put/seqid=0 2024-12-05T12:15:13,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742212_1388 (size=12151) 2024-12-05T12:15:13,030 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/d4be887797684392ae0909ad400f4da0 2024-12-05T12:15:13,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/bc0a2a9fc68745f1a70e8584dc8f0c9e is 50, key is test_row_0/C:col10/1733400910825/Put/seqid=0 2024-12-05T12:15:13,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400973050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400973051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742213_1389 (size=12151) 2024-12-05T12:15:13,072 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/bc0a2a9fc68745f1a70e8584dc8f0c9e 2024-12-05T12:15:13,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400973061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400973065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/dfbe4aafd13f4875a8546ea9b9ac9d4b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/dfbe4aafd13f4875a8546ea9b9ac9d4b 2024-12-05T12:15:13,083 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/dfbe4aafd13f4875a8546ea9b9ac9d4b, entries=150, sequenceid=158, filesize=30.4 K 2024-12-05T12:15:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T12:15:13,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/d4be887797684392ae0909ad400f4da0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/d4be887797684392ae0909ad400f4da0 2024-12-05T12:15:13,088 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/d4be887797684392ae0909ad400f4da0, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T12:15:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/bc0a2a9fc68745f1a70e8584dc8f0c9e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bc0a2a9fc68745f1a70e8584dc8f0c9e 2024-12-05T12:15:13,094 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bc0a2a9fc68745f1a70e8584dc8f0c9e, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T12:15:13,095 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8ac3794477a85e9bc205a48eed1522fa in 177ms, sequenceid=158, compaction requested=true 2024-12-05T12:15:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-05T12:15:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-05T12:15:13,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-05T12:15:13,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 332 msec 2024-12-05T12:15:13,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 337 msec 2024-12-05T12:15:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:13,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T12:15:13,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:13,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:13,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:13,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:13,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:13,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:13,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120506e5282357664f59aefa4eea271c799b_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400913061/Put/seqid=0 2024-12-05T12:15:13,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742214_1390 (size=17284) 2024-12-05T12:15:13,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400973206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400973206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400973207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400973214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400973323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400973323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400973324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400973324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T12:15:13,393 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:13,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-05T12:15:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T12:15:13,395 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:13,395 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:13,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T12:15:13,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T12:15:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:13,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:13,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400973543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400973543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400973543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400973543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:13,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120506e5282357664f59aefa4eea271c799b_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120506e5282357664f59aefa4eea271c799b_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:13,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/f429d5d4b5ed4ceb83922510250dcb29, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:13,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/f429d5d4b5ed4ceb83922510250dcb29 is 175, key is test_row_0/A:col10/1733400913061/Put/seqid=0 2024-12-05T12:15:13,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742215_1391 (size=48389) 2024-12-05T12:15:13,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T12:15:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:13,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T12:15:13,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T12:15:13,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:13,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:13,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:13,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:13,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:13,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400973863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400973863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400973863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400973863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:14,006 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/f429d5d4b5ed4ceb83922510250dcb29 2024-12-05T12:15:14,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T12:15:14,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:14,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/872a8f8422d147039682febf71afff51 is 50, key is test_row_0/B:col10/1733400913061/Put/seqid=0 2024-12-05T12:15:14,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742216_1392 (size=12151) 2024-12-05T12:15:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T12:15:14,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T12:15:14,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:14,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T12:15:14,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:14,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400974373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:14,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400974373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:14,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400974373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:14,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400974373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:14,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/872a8f8422d147039682febf71afff51 2024-12-05T12:15:14,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/333eb64cb8f242a7a52a33c974f4c273 is 50, key is test_row_0/C:col10/1733400913061/Put/seqid=0 2024-12-05T12:15:14,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742217_1393 (size=12151) 2024-12-05T12:15:14,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T12:15:14,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:14,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,516 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T12:15:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T12:15:14,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T12:15:14,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:14,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T12:15:14,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:14,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:14,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/333eb64cb8f242a7a52a33c974f4c273 2024-12-05T12:15:14,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/f429d5d4b5ed4ceb83922510250dcb29 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/f429d5d4b5ed4ceb83922510250dcb29 2024-12-05T12:15:14,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/f429d5d4b5ed4ceb83922510250dcb29, entries=250, sequenceid=176, filesize=47.3 K 2024-12-05T12:15:14,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/872a8f8422d147039682febf71afff51 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/872a8f8422d147039682febf71afff51 2024-12-05T12:15:14,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/872a8f8422d147039682febf71afff51, entries=150, sequenceid=176, filesize=11.9 K 2024-12-05T12:15:14,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/333eb64cb8f242a7a52a33c974f4c273 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/333eb64cb8f242a7a52a33c974f4c273 2024-12-05T12:15:14,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/333eb64cb8f242a7a52a33c974f4c273, entries=150, sequenceid=176, filesize=11.9 K 2024-12-05T12:15:14,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8ac3794477a85e9bc205a48eed1522fa in 1665ms, sequenceid=176, compaction requested=true 2024-12-05T12:15:14,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:14,844 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:15:14,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:14,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:14,844 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:15:14,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:14,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:14,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:14,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:14,846 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:15:14,846 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:15:14,846 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/B is initiating minor compaction (all files) 2024-12-05T12:15:14,846 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/A is initiating minor compaction (all files) 2024-12-05T12:15:14,846 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/A in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,846 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/B in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,846 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/456ae25751b6408ca4b3fe6e7164f359, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/43daeeeb54f246ad963de4a2f0643b07, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/d4be887797684392ae0909ad400f4da0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/872a8f8422d147039682febf71afff51] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=47.6 K 2024-12-05T12:15:14,846 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/754bd9a5a8aa46e7ba6d92d57b0090ee, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/317300866d8444419d95989f7d07e592, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/dfbe4aafd13f4875a8546ea9b9ac9d4b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/f429d5d4b5ed4ceb83922510250dcb29] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=138.5 K 2024-12-05T12:15:14,847 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,847 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/754bd9a5a8aa46e7ba6d92d57b0090ee, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/317300866d8444419d95989f7d07e592, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/dfbe4aafd13f4875a8546ea9b9ac9d4b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/f429d5d4b5ed4ceb83922510250dcb29] 2024-12-05T12:15:14,850 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 456ae25751b6408ca4b3fe6e7164f359, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733400908604 2024-12-05T12:15:14,850 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 754bd9a5a8aa46e7ba6d92d57b0090ee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733400908604 2024-12-05T12:15:14,851 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 317300866d8444419d95989f7d07e592, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733400908645 2024-12-05T12:15:14,851 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 43daeeeb54f246ad963de4a2f0643b07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733400908645 2024-12-05T12:15:14,851 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfbe4aafd13f4875a8546ea9b9ac9d4b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733400910825 2024-12-05T12:15:14,851 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d4be887797684392ae0909ad400f4da0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733400910825 2024-12-05T12:15:14,852 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting f429d5d4b5ed4ceb83922510250dcb29, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733400913043 2024-12-05T12:15:14,852 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 872a8f8422d147039682febf71afff51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733400913061 2024-12-05T12:15:14,861 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:14,865 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120501f6fee54e4f45fb9b8eed4e3ee59c3b_8ac3794477a85e9bc205a48eed1522fa store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:14,866 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#B#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:14,867 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/821abe8c2409420088c1060fcc0d9f33 is 50, key is test_row_0/B:col10/1733400913061/Put/seqid=0 2024-12-05T12:15:14,868 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120501f6fee54e4f45fb9b8eed4e3ee59c3b_8ac3794477a85e9bc205a48eed1522fa, store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:14,868 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120501f6fee54e4f45fb9b8eed4e3ee59c3b_8ac3794477a85e9bc205a48eed1522fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:14,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742218_1394 (size=12595) 2024-12-05T12:15:14,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742219_1395 (size=4469) 2024-12-05T12:15:14,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T12:15:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:14,924 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T12:15:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:14,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ab0e72df50f547a78d56cc94ca802c4e_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400913213/Put/seqid=0 2024-12-05T12:15:14,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742220_1396 (size=12304) 2024-12-05T12:15:15,280 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:15,281 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/a443b45af0e641a399eb56632c3fdb5e is 175, key is test_row_0/A:col10/1733400913061/Put/seqid=0 2024-12-05T12:15:15,283 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/821abe8c2409420088c1060fcc0d9f33 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/821abe8c2409420088c1060fcc0d9f33 2024-12-05T12:15:15,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742221_1397 (size=31549) 2024-12-05T12:15:15,288 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/B of 8ac3794477a85e9bc205a48eed1522fa into 821abe8c2409420088c1060fcc0d9f33(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:15,288 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:15,288 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/B, priority=12, startTime=1733400914844; duration=0sec 2024-12-05T12:15:15,288 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:15,288 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:B 2024-12-05T12:15:15,288 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:15:15,289 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:15:15,289 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/C is initiating minor compaction (all files) 2024-12-05T12:15:15,289 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/C in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:15,289 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/92f3eea0048a42909be6c5acb95ac974, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/5f6e6a9ec1ad4f0384c46ff9ceb13c77, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bc0a2a9fc68745f1a70e8584dc8f0c9e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/333eb64cb8f242a7a52a33c974f4c273] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=47.6 K 2024-12-05T12:15:15,290 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f3eea0048a42909be6c5acb95ac974, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733400908604 2024-12-05T12:15:15,290 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6e6a9ec1ad4f0384c46ff9ceb13c77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733400908645 2024-12-05T12:15:15,290 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting bc0a2a9fc68745f1a70e8584dc8f0c9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733400910825 2024-12-05T12:15:15,290 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 333eb64cb8f242a7a52a33c974f4c273, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733400913061 2024-12-05T12:15:15,296 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:15,296 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/d25ed2f2805a45ddb58cbc54b1041a76 is 50, key is test_row_0/C:col10/1733400913061/Put/seqid=0 2024-12-05T12:15:15,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742222_1398 (size=12595) 2024-12-05T12:15:15,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:15,337 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ab0e72df50f547a78d56cc94ca802c4e_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ab0e72df50f547a78d56cc94ca802c4e_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:15,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/adb235bf5b404157a18c8b6c309801f4, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:15,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/adb235bf5b404157a18c8b6c309801f4 is 175, key is test_row_0/A:col10/1733400913213/Put/seqid=0 2024-12-05T12:15:15,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742223_1399 (size=31105) 2024-12-05T12:15:15,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:15,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400975418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400975418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400975419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400975420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400975543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400975543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400975543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400975543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T12:15:15,688 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/a443b45af0e641a399eb56632c3fdb5e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/a443b45af0e641a399eb56632c3fdb5e 2024-12-05T12:15:15,693 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/A of 8ac3794477a85e9bc205a48eed1522fa into a443b45af0e641a399eb56632c3fdb5e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:15,693 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:15,693 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/A, priority=12, startTime=1733400914844; duration=0sec 2024-12-05T12:15:15,693 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:15,693 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:A 2024-12-05T12:15:15,704 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/d25ed2f2805a45ddb58cbc54b1041a76 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d25ed2f2805a45ddb58cbc54b1041a76 2024-12-05T12:15:15,708 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/C of 8ac3794477a85e9bc205a48eed1522fa into d25ed2f2805a45ddb58cbc54b1041a76(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:15,708 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:15,708 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/C, priority=12, startTime=1733400914845; duration=0sec 2024-12-05T12:15:15,708 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:15,708 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:C 2024-12-05T12:15:15,745 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/adb235bf5b404157a18c8b6c309801f4 2024-12-05T12:15:15,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/15c8244da51945208246010a3a5494e6 is 50, key is test_row_0/B:col10/1733400913213/Put/seqid=0 2024-12-05T12:15:15,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400975753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400975754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400975754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400975754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:15,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742224_1400 (size=12151) 2024-12-05T12:15:16,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400976062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:16,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400976072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400976073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400976073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:16,167 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/15c8244da51945208246010a3a5494e6 2024-12-05T12:15:16,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/36fb2c4fdbcd47b38518932ad00d4aa2 is 50, key is test_row_0/C:col10/1733400913213/Put/seqid=0 2024-12-05T12:15:16,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742225_1401 (size=12151) 2024-12-05T12:15:16,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400976572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:16,584 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/36fb2c4fdbcd47b38518932ad00d4aa2 2024-12-05T12:15:16,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400976583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:16,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400976583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:16,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/adb235bf5b404157a18c8b6c309801f4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/adb235bf5b404157a18c8b6c309801f4 2024-12-05T12:15:16,592 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/adb235bf5b404157a18c8b6c309801f4, entries=150, sequenceid=190, filesize=30.4 K 2024-12-05T12:15:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/15c8244da51945208246010a3a5494e6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/15c8244da51945208246010a3a5494e6 2024-12-05T12:15:16,599 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/15c8244da51945208246010a3a5494e6, entries=150, sequenceid=190, filesize=11.9 K 2024-12-05T12:15:16,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400976593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:16,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/36fb2c4fdbcd47b38518932ad00d4aa2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/36fb2c4fdbcd47b38518932ad00d4aa2 2024-12-05T12:15:16,604 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/36fb2c4fdbcd47b38518932ad00d4aa2, entries=150, sequenceid=190, filesize=11.9 K 2024-12-05T12:15:16,605 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8ac3794477a85e9bc205a48eed1522fa in 1681ms, sequenceid=190, compaction requested=false 2024-12-05T12:15:16,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:16,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:16,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-05T12:15:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-05T12:15:16,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-05T12:15:16,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2100 sec 2024-12-05T12:15:16,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.2150 sec 2024-12-05T12:15:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T12:15:17,574 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:17,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-05T12:15:17,576 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T12:15:17,583 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:17,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:17,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T12:15:17,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:17,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:17,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:17,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:17,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:17,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:17,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120544622073c19b4396a50f2f5eefa970a6_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400915418/Put/seqid=0 2024-12-05T12:15:17,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400977628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400977629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400977630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742226_1402 (size=14794) 2024-12-05T12:15:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400977640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T12:15:17,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-05T12:15:17,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:17,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:17,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:17,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:17,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:17,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400977753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400977753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400977753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400977753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-05T12:15:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:17,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T12:15:17,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400977972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400977973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400977973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:17,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400977973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-05T12:15:18,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:18,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,044 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:18,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120544622073c19b4396a50f2f5eefa970a6_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120544622073c19b4396a50f2f5eefa970a6_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:18,049 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/05e85431a91e421a98f17e6fbe711744, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:18,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/05e85431a91e421a98f17e6fbe711744 is 175, key is test_row_0/A:col10/1733400915418/Put/seqid=0 2024-12-05T12:15:18,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742227_1403 (size=39749) 2024-12-05T12:15:18,090 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/05e85431a91e421a98f17e6fbe711744 2024-12-05T12:15:18,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/2a0244db039a433fb16b304995ae2b9f is 50, key is test_row_0/B:col10/1733400915418/Put/seqid=0 2024-12-05T12:15:18,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742228_1404 (size=12151) 2024-12-05T12:15:18,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/2a0244db039a433fb16b304995ae2b9f 2024-12-05T12:15:18,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/f15ed24362b947a3aed45fa9ffa24463 is 50, key is test_row_0/C:col10/1733400915418/Put/seqid=0 2024-12-05T12:15:18,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742229_1405 (size=12151) 2024-12-05T12:15:18,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-05T12:15:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T12:15:18,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400978293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400978293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400978293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400978293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-05T12:15:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-05T12:15:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:18,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/f15ed24362b947a3aed45fa9ffa24463 2024-12-05T12:15:18,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/05e85431a91e421a98f17e6fbe711744 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/05e85431a91e421a98f17e6fbe711744 2024-12-05T12:15:18,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/05e85431a91e421a98f17e6fbe711744, entries=200, sequenceid=212, filesize=38.8 K 2024-12-05T12:15:18,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/2a0244db039a433fb16b304995ae2b9f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/2a0244db039a433fb16b304995ae2b9f 2024-12-05T12:15:18,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/2a0244db039a433fb16b304995ae2b9f, entries=150, sequenceid=212, filesize=11.9 K 2024-12-05T12:15:18,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/f15ed24362b947a3aed45fa9ffa24463 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/f15ed24362b947a3aed45fa9ffa24463 2024-12-05T12:15:18,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/f15ed24362b947a3aed45fa9ffa24463, entries=150, sequenceid=212, filesize=11.9 K 2024-12-05T12:15:18,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8ac3794477a85e9bc205a48eed1522fa in 1035ms, sequenceid=212, compaction requested=true 2024-12-05T12:15:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:18,627 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:18,627 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:18,628 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:18,628 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:18,628 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/A is initiating minor compaction (all files) 2024-12-05T12:15:18,628 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/B is initiating minor compaction (all files) 2024-12-05T12:15:18,628 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/A in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,628 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/B in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,628 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/a443b45af0e641a399eb56632c3fdb5e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/adb235bf5b404157a18c8b6c309801f4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/05e85431a91e421a98f17e6fbe711744] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=100.0 K 2024-12-05T12:15:18,628 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/821abe8c2409420088c1060fcc0d9f33, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/15c8244da51945208246010a3a5494e6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/2a0244db039a433fb16b304995ae2b9f] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=36.0 K 2024-12-05T12:15:18,628 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,628 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/a443b45af0e641a399eb56632c3fdb5e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/adb235bf5b404157a18c8b6c309801f4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/05e85431a91e421a98f17e6fbe711744] 2024-12-05T12:15:18,628 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 821abe8c2409420088c1060fcc0d9f33, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733400913061 2024-12-05T12:15:18,628 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting a443b45af0e641a399eb56632c3fdb5e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733400913061 2024-12-05T12:15:18,628 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c8244da51945208246010a3a5494e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733400913204 2024-12-05T12:15:18,629 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0244db039a433fb16b304995ae2b9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733400915416 2024-12-05T12:15:18,629 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb235bf5b404157a18c8b6c309801f4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733400913204 2024-12-05T12:15:18,630 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05e85431a91e421a98f17e6fbe711744, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733400915416 2024-12-05T12:15:18,638 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:18,638 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/7ef74a524f9247fbb5f17fa70034856f is 50, key is test_row_0/B:col10/1733400915418/Put/seqid=0 2024-12-05T12:15:18,643 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:18,648 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205d81807ed7f0841289bc80a125da7526f_8ac3794477a85e9bc205a48eed1522fa store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:18,649 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205d81807ed7f0841289bc80a125da7526f_8ac3794477a85e9bc205a48eed1522fa, store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:18,649 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d81807ed7f0841289bc80a125da7526f_8ac3794477a85e9bc205a48eed1522fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:18,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-05T12:15:18,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:18,666 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T12:15:18,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:18,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:18,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:18,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:18,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:18,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:18,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742230_1406 (size=12697) 2024-12-05T12:15:18,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051b9ea33f6370460fab1dd7078b649a87_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400917626/Put/seqid=0 2024-12-05T12:15:18,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742231_1407 (size=4469) 2024-12-05T12:15:18,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742232_1408 (size=12304) 2024-12-05T12:15:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:18,711 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051b9ea33f6370460fab1dd7078b649a87_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051b9ea33f6370460fab1dd7078b649a87_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:18,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/13bb44384edb4d428a01729b10b3173b, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:18,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/13bb44384edb4d428a01729b10b3173b is 175, key is test_row_0/A:col10/1733400917626/Put/seqid=0 2024-12-05T12:15:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T12:15:18,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742233_1409 (size=31105) 2024-12-05T12:15:18,728 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/13bb44384edb4d428a01729b10b3173b 2024-12-05T12:15:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/b909d8352ec24b1289543cd27d8e6392 is 50, key is test_row_0/B:col10/1733400917626/Put/seqid=0 2024-12-05T12:15:18,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742234_1410 (size=12151) 2024-12-05T12:15:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:18,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:18,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400978850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400978859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400978863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400978863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400978973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400978983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400978983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400978983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:19,076 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/7ef74a524f9247fbb5f17fa70034856f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/7ef74a524f9247fbb5f17fa70034856f 2024-12-05T12:15:19,083 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/B of 8ac3794477a85e9bc205a48eed1522fa into 7ef74a524f9247fbb5f17fa70034856f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:19,083 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:19,083 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/B, priority=13, startTime=1733400918627; duration=0sec 2024-12-05T12:15:19,083 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:19,083 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:B 2024-12-05T12:15:19,083 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:19,084 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:19,084 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 8ac3794477a85e9bc205a48eed1522fa/C is initiating minor compaction (all files) 2024-12-05T12:15:19,084 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ac3794477a85e9bc205a48eed1522fa/C in TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:19,084 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d25ed2f2805a45ddb58cbc54b1041a76, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/36fb2c4fdbcd47b38518932ad00d4aa2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/f15ed24362b947a3aed45fa9ffa24463] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp, totalSize=36.0 K 2024-12-05T12:15:19,085 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d25ed2f2805a45ddb58cbc54b1041a76, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733400913061 2024-12-05T12:15:19,085 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 36fb2c4fdbcd47b38518932ad00d4aa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733400913204 2024-12-05T12:15:19,085 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting f15ed24362b947a3aed45fa9ffa24463, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733400915416 2024-12-05T12:15:19,094 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#C#compaction#343 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:19,094 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ac3794477a85e9bc205a48eed1522fa#A#compaction#340 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:19,095 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/636229e1e4b849bd88cd1af7b137f591 is 50, key is test_row_0/C:col10/1733400915418/Put/seqid=0 2024-12-05T12:15:19,095 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/341bdafcd57745dd9cab280afe8cc6f3 is 175, key is test_row_0/A:col10/1733400915418/Put/seqid=0 2024-12-05T12:15:19,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742236_1412 (size=31651) 2024-12-05T12:15:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742235_1411 (size=12697) 2024-12-05T12:15:19,110 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/341bdafcd57745dd9cab280afe8cc6f3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/341bdafcd57745dd9cab280afe8cc6f3 2024-12-05T12:15:19,117 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/A of 8ac3794477a85e9bc205a48eed1522fa into 341bdafcd57745dd9cab280afe8cc6f3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:19,117 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:19,117 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/A, priority=13, startTime=1733400918627; duration=0sec 2024-12-05T12:15:19,117 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:19,117 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:A 2024-12-05T12:15:19,118 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/636229e1e4b849bd88cd1af7b137f591 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/636229e1e4b849bd88cd1af7b137f591 2024-12-05T12:15:19,124 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ac3794477a85e9bc205a48eed1522fa/C of 8ac3794477a85e9bc205a48eed1522fa into 636229e1e4b849bd88cd1af7b137f591(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:19,124 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:19,124 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., storeName=8ac3794477a85e9bc205a48eed1522fa/C, priority=13, startTime=1733400918627; duration=0sec 2024-12-05T12:15:19,124 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:19,125 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:C 2024-12-05T12:15:19,169 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/b909d8352ec24b1289543cd27d8e6392 2024-12-05T12:15:19,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/9452aa6b7d2f4e63a89d76421269d57c is 50, key is test_row_0/C:col10/1733400917626/Put/seqid=0 2024-12-05T12:15:19,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742237_1413 (size=12151) 2024-12-05T12:15:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400979193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400979193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400979194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400979194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:19,378 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/72c3cd5245a8405290c2efd4a6171a5c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/c8575b3a1078407699594fbb1b3def94, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/5a92e7c41abe435ea12b35f495a21936, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/8ae33c487d9d451895d7a26ecfc36656, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/1e0685a9aab045a8a38fa8f34ac47262, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/18d19aba699c4d82bd998fb2ade6fcf2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/3d1b15dc0ccb44fca7ff779002debb5b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/754bd9a5a8aa46e7ba6d92d57b0090ee, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/9fb37f360aef460a9a2fa5bf3b8af658, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/317300866d8444419d95989f7d07e592, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/dfbe4aafd13f4875a8546ea9b9ac9d4b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/f429d5d4b5ed4ceb83922510250dcb29, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/a443b45af0e641a399eb56632c3fdb5e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/adb235bf5b404157a18c8b6c309801f4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/05e85431a91e421a98f17e6fbe711744] to archive 2024-12-05T12:15:19,379 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:15:19,381 WARN [regionserver/9ea9abad0e9b:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-05T12:15:19,382 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/72c3cd5245a8405290c2efd4a6171a5c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/72c3cd5245a8405290c2efd4a6171a5c 2024-12-05T12:15:19,383 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/c8575b3a1078407699594fbb1b3def94 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/c8575b3a1078407699594fbb1b3def94 2024-12-05T12:15:19,386 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/5a92e7c41abe435ea12b35f495a21936 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/5a92e7c41abe435ea12b35f495a21936 2024-12-05T12:15:19,388 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/8ae33c487d9d451895d7a26ecfc36656 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/8ae33c487d9d451895d7a26ecfc36656 2024-12-05T12:15:19,389 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ddcc48ca4c8a40a59f3fa6a8f33ea10d 2024-12-05T12:15:19,390 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/1e0685a9aab045a8a38fa8f34ac47262 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/1e0685a9aab045a8a38fa8f34ac47262 2024-12-05T12:15:19,391 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/18d19aba699c4d82bd998fb2ade6fcf2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/18d19aba699c4d82bd998fb2ade6fcf2 2024-12-05T12:15:19,393 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/3d1b15dc0ccb44fca7ff779002debb5b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/3d1b15dc0ccb44fca7ff779002debb5b 2024-12-05T12:15:19,395 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/754bd9a5a8aa46e7ba6d92d57b0090ee to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/754bd9a5a8aa46e7ba6d92d57b0090ee 2024-12-05T12:15:19,396 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/9fb37f360aef460a9a2fa5bf3b8af658 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/9fb37f360aef460a9a2fa5bf3b8af658 2024-12-05T12:15:19,397 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/317300866d8444419d95989f7d07e592 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/317300866d8444419d95989f7d07e592 2024-12-05T12:15:19,398 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/dfbe4aafd13f4875a8546ea9b9ac9d4b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/dfbe4aafd13f4875a8546ea9b9ac9d4b 2024-12-05T12:15:19,401 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/f429d5d4b5ed4ceb83922510250dcb29 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/f429d5d4b5ed4ceb83922510250dcb29 2024-12-05T12:15:19,402 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/a443b45af0e641a399eb56632c3fdb5e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/a443b45af0e641a399eb56632c3fdb5e 2024-12-05T12:15:19,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/adb235bf5b404157a18c8b6c309801f4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/adb235bf5b404157a18c8b6c309801f4 2024-12-05T12:15:19,406 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/05e85431a91e421a98f17e6fbe711744 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/05e85431a91e421a98f17e6fbe711744 2024-12-05T12:15:19,408 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/94841b45ec3443a78463550e37ae52bf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/c98c37b0fe33411a94b5d0c43c86bddc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/ee087b1aaa7c413a898faace499ff00a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/286002a02e09455eb76f6c6be23a9260, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/4ec00d1b785d4e54b085258c8ecd1b36, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/8332d4d5a5454d34a14bb95f8c7bc381, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/0e6f3dfbaaf841f293b3c28e63a02357, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/421d6d2f88664886866bd69fa13d3db5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/456ae25751b6408ca4b3fe6e7164f359, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/60ea64a005004759b8f809ad005bc8cc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/43daeeeb54f246ad963de4a2f0643b07, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/d4be887797684392ae0909ad400f4da0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/821abe8c2409420088c1060fcc0d9f33, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/872a8f8422d147039682febf71afff51, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/15c8244da51945208246010a3a5494e6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/2a0244db039a433fb16b304995ae2b9f] to archive 2024-12-05T12:15:19,409 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:15:19,410 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/94841b45ec3443a78463550e37ae52bf to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/94841b45ec3443a78463550e37ae52bf 2024-12-05T12:15:19,411 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/c98c37b0fe33411a94b5d0c43c86bddc to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/c98c37b0fe33411a94b5d0c43c86bddc 2024-12-05T12:15:19,412 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/ee087b1aaa7c413a898faace499ff00a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/ee087b1aaa7c413a898faace499ff00a 2024-12-05T12:15:19,413 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/286002a02e09455eb76f6c6be23a9260 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/286002a02e09455eb76f6c6be23a9260 2024-12-05T12:15:19,414 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/4ec00d1b785d4e54b085258c8ecd1b36 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/4ec00d1b785d4e54b085258c8ecd1b36 2024-12-05T12:15:19,414 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/8332d4d5a5454d34a14bb95f8c7bc381 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/8332d4d5a5454d34a14bb95f8c7bc381 2024-12-05T12:15:19,415 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/0e6f3dfbaaf841f293b3c28e63a02357 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/0e6f3dfbaaf841f293b3c28e63a02357 2024-12-05T12:15:19,416 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/421d6d2f88664886866bd69fa13d3db5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/421d6d2f88664886866bd69fa13d3db5 2024-12-05T12:15:19,416 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/456ae25751b6408ca4b3fe6e7164f359 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/456ae25751b6408ca4b3fe6e7164f359 2024-12-05T12:15:19,417 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/60ea64a005004759b8f809ad005bc8cc to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/60ea64a005004759b8f809ad005bc8cc 2024-12-05T12:15:19,418 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/43daeeeb54f246ad963de4a2f0643b07 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/43daeeeb54f246ad963de4a2f0643b07 2024-12-05T12:15:19,418 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/d4be887797684392ae0909ad400f4da0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/d4be887797684392ae0909ad400f4da0 2024-12-05T12:15:19,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/821abe8c2409420088c1060fcc0d9f33 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/821abe8c2409420088c1060fcc0d9f33 2024-12-05T12:15:19,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/872a8f8422d147039682febf71afff51 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/872a8f8422d147039682febf71afff51 2024-12-05T12:15:19,420 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/15c8244da51945208246010a3a5494e6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/15c8244da51945208246010a3a5494e6 2024-12-05T12:15:19,421 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/2a0244db039a433fb16b304995ae2b9f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/2a0244db039a433fb16b304995ae2b9f 2024-12-05T12:15:19,423 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d6437dd916224eb8872a7965cab1ed24, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/ebe5c2fc71394a288f4195dd0be9e7e7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/71c02ffcaf4e4cda928b8935b974c9da, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/207a697343314feba50911417627662a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bec96ee7803e4c079fb4ef8c0ff3075a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/2f188a71e253445b9af1c86556d8de6e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d29995ca9a0b4161b62dcd6047338eef, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/0454fdad311d4e8ab27a2871e48d8ed7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/92f3eea0048a42909be6c5acb95ac974, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/c91b4f61258444be87cf1e7a89aed3ed, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/5f6e6a9ec1ad4f0384c46ff9ceb13c77, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bc0a2a9fc68745f1a70e8584dc8f0c9e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d25ed2f2805a45ddb58cbc54b1041a76, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/333eb64cb8f242a7a52a33c974f4c273, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/36fb2c4fdbcd47b38518932ad00d4aa2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/f15ed24362b947a3aed45fa9ffa24463] to archive 2024-12-05T12:15:19,424 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:15:19,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d6437dd916224eb8872a7965cab1ed24 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d6437dd916224eb8872a7965cab1ed24 2024-12-05T12:15:19,426 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/ebe5c2fc71394a288f4195dd0be9e7e7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/ebe5c2fc71394a288f4195dd0be9e7e7 2024-12-05T12:15:19,426 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/71c02ffcaf4e4cda928b8935b974c9da to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/71c02ffcaf4e4cda928b8935b974c9da 2024-12-05T12:15:19,427 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/207a697343314feba50911417627662a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/207a697343314feba50911417627662a 2024-12-05T12:15:19,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bec96ee7803e4c079fb4ef8c0ff3075a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bec96ee7803e4c079fb4ef8c0ff3075a 2024-12-05T12:15:19,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/2f188a71e253445b9af1c86556d8de6e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/2f188a71e253445b9af1c86556d8de6e 2024-12-05T12:15:19,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d29995ca9a0b4161b62dcd6047338eef to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d29995ca9a0b4161b62dcd6047338eef 2024-12-05T12:15:19,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/0454fdad311d4e8ab27a2871e48d8ed7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/0454fdad311d4e8ab27a2871e48d8ed7 2024-12-05T12:15:19,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/92f3eea0048a42909be6c5acb95ac974 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/92f3eea0048a42909be6c5acb95ac974 2024-12-05T12:15:19,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/c91b4f61258444be87cf1e7a89aed3ed to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/c91b4f61258444be87cf1e7a89aed3ed 2024-12-05T12:15:19,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/5f6e6a9ec1ad4f0384c46ff9ceb13c77 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/5f6e6a9ec1ad4f0384c46ff9ceb13c77 2024-12-05T12:15:19,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bc0a2a9fc68745f1a70e8584dc8f0c9e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/bc0a2a9fc68745f1a70e8584dc8f0c9e 2024-12-05T12:15:19,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d25ed2f2805a45ddb58cbc54b1041a76 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/d25ed2f2805a45ddb58cbc54b1041a76 2024-12-05T12:15:19,434 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/333eb64cb8f242a7a52a33c974f4c273 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/333eb64cb8f242a7a52a33c974f4c273 2024-12-05T12:15:19,435 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/36fb2c4fdbcd47b38518932ad00d4aa2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/36fb2c4fdbcd47b38518932ad00d4aa2 2024-12-05T12:15:19,436 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/f15ed24362b947a3aed45fa9ffa24463 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/f15ed24362b947a3aed45fa9ffa24463 2024-12-05T12:15:19,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400979503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:19,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400979503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:19,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400979503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:19,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400979504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:19,580 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/9452aa6b7d2f4e63a89d76421269d57c 2024-12-05T12:15:19,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/13bb44384edb4d428a01729b10b3173b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/13bb44384edb4d428a01729b10b3173b 2024-12-05T12:15:19,587 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/13bb44384edb4d428a01729b10b3173b, entries=150, sequenceid=225, filesize=30.4 K 2024-12-05T12:15:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/b909d8352ec24b1289543cd27d8e6392 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/b909d8352ec24b1289543cd27d8e6392 2024-12-05T12:15:19,590 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/b909d8352ec24b1289543cd27d8e6392, entries=150, sequenceid=225, filesize=11.9 K 2024-12-05T12:15:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/9452aa6b7d2f4e63a89d76421269d57c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/9452aa6b7d2f4e63a89d76421269d57c 2024-12-05T12:15:19,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/9452aa6b7d2f4e63a89d76421269d57c, entries=150, sequenceid=225, filesize=11.9 K 2024-12-05T12:15:19,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8ac3794477a85e9bc205a48eed1522fa in 928ms, sequenceid=225, compaction requested=false 2024-12-05T12:15:19,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:19,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:19,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-05T12:15:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-05T12:15:19,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-05T12:15:19,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0130 sec 2024-12-05T12:15:19,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.0240 sec 2024-12-05T12:15:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T12:15:19,733 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:19,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-05T12:15:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T12:15:19,738 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:19,740 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:19,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T12:15:19,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-05T12:15:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:19,895 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T12:15:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:19,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120529cb0d88b4144361a2dbd9de3c8f66d6_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400918860/Put/seqid=0 2024-12-05T12:15:19,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742238_1414 (size=12304) 2024-12-05T12:15:19,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:19,974 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120529cb0d88b4144361a2dbd9de3c8f66d6_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120529cb0d88b4144361a2dbd9de3c8f66d6_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:19,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ca53347e024a488fb9cec9b63ea8745a, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:19,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ca53347e024a488fb9cec9b63ea8745a is 175, key is test_row_0/A:col10/1733400918860/Put/seqid=0 2024-12-05T12:15:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:20,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. as already flushing 2024-12-05T12:15:20,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742239_1415 (size=31105) 2024-12-05T12:15:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400980038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400980040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400980043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400980044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T12:15:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,136 INFO [Thread-1570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1570. 2024-12-05T12:15:20,137 DEBUG [Thread-1570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:20,137 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:20,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:20,139 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:20,141 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-12-05T12:15:20,141 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:20,141 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:20,141 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:20,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:20,143 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-12-05T12:15:20,143 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:20,143 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:20,143 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:20,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:20,146 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-12-05T12:15:20,146 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:20,146 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:20,146 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:20,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:20,151 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-12-05T12:15:20,151 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:20,151 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:20,151 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:20,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400980153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400980153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400980153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400980163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T12:15:20,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400980363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400980363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400980363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400980372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,440 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ca53347e024a488fb9cec9b63ea8745a 2024-12-05T12:15:20,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/614a3fcf4d334b9e9fb249d6e17c8683 is 50, key is test_row_0/B:col10/1733400918860/Put/seqid=0 2024-12-05T12:15:20,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742240_1416 (size=12151) 2024-12-05T12:15:20,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400980673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400980673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400980673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400980683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:20,851 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/614a3fcf4d334b9e9fb249d6e17c8683 2024-12-05T12:15:20,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/55ce2cb10352470282b3d0974bb419b8 is 50, key is test_row_0/C:col10/1733400918860/Put/seqid=0 2024-12-05T12:15:20,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742241_1417 (size=12151) 2024-12-05T12:15:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T12:15:21,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:21,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:21,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53548 deadline: 1733400981182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733400981182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53560 deadline: 1733400981182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:21,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53598 deadline: 1733400981193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:21,260 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/55ce2cb10352470282b3d0974bb419b8 2024-12-05T12:15:21,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ca53347e024a488fb9cec9b63ea8745a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ca53347e024a488fb9cec9b63ea8745a 2024-12-05T12:15:21,268 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ca53347e024a488fb9cec9b63ea8745a, entries=150, sequenceid=247, filesize=30.4 K 2024-12-05T12:15:21,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/614a3fcf4d334b9e9fb249d6e17c8683 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/614a3fcf4d334b9e9fb249d6e17c8683 2024-12-05T12:15:21,273 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/614a3fcf4d334b9e9fb249d6e17c8683, entries=150, sequenceid=247, filesize=11.9 K 2024-12-05T12:15:21,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/55ce2cb10352470282b3d0974bb419b8 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/55ce2cb10352470282b3d0974bb419b8 2024-12-05T12:15:21,278 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/55ce2cb10352470282b3d0974bb419b8, entries=150, sequenceid=247, filesize=11.9 K 2024-12-05T12:15:21,278 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8ac3794477a85e9bc205a48eed1522fa in 1383ms, sequenceid=247, compaction requested=true 2024-12-05T12:15:21,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:21,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:21,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-05T12:15:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-05T12:15:21,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-05T12:15:21,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5400 sec 2024-12-05T12:15:21,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.5480 sec 2024-12-05T12:15:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:21,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T12:15:21,365 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-12-05T12:15:21,365 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:21,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:21,365 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:21,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:21,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:21,365 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:21,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:21,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:21,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:21,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:21,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120555533270071f453c967ad5771485eacd_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_0/A:col10/1733400920043/Put/seqid=0 2024-12-05T12:15:21,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742242_1418 (size=12454) 2024-12-05T12:15:21,775 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:21,779 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120555533270071f453c967ad5771485eacd_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120555533270071f453c967ad5771485eacd_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:21,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ba0cc2830f76435a9d71b0e1a173d214, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:21,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ba0cc2830f76435a9d71b0e1a173d214 is 175, key is test_row_0/A:col10/1733400920043/Put/seqid=0 2024-12-05T12:15:21,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742243_1419 (size=31255) 2024-12-05T12:15:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T12:15:21,893 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:22,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ba0cc2830f76435a9d71b0e1a173d214 2024-12-05T12:15:22,197 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-12-05T12:15:22,197 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-12-05T12:15:22,197 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:22,197 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:22,197 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:22,197 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:22,197 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:22,197 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:22,197 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:22,197 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:22,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/405058246a8245deac51e51a3c600bdd is 50, key is test_row_0/B:col10/1733400920043/Put/seqid=0 2024-12-05T12:15:22,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742244_1420 (size=12301) 2024-12-05T12:15:22,205 INFO [Thread-1561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1561. 2024-12-05T12:15:22,205 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-12-05T12:15:22,205 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:22,205 DEBUG [Thread-1561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:22,205 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:22,205 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:22,205 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:22,205 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:22,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:22,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1597 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4790 rows 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1594 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4777 rows 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1597 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4790 rows 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1597 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4790 rows 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1590 2024-12-05T12:15:22,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4768 rows 2024-12-05T12:15:22,206 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:15:22,206 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:15:22,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:22,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:22,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:22,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T12:15:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T12:15:22,211 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400922211"}]},"ts":"1733400922211"} 2024-12-05T12:15:22,213 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T12:15:22,213 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T12:15:22,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T12:15:22,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, UNASSIGN}] 2024-12-05T12:15:22,217 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, UNASSIGN 2024-12-05T12:15:22,218 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=8ac3794477a85e9bc205a48eed1522fa, regionState=CLOSING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:22,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, UNASSIGN because future has completed 2024-12-05T12:15:22,219 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T12:15:22,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T12:15:22,373 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:22,373 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T12:15:22,374 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 8ac3794477a85e9bc205a48eed1522fa, disabling compactions & flushes 2024-12-05T12:15:22,374 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T12:15:22,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/405058246a8245deac51e51a3c600bdd 2024-12-05T12:15:22,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/ebb3ec44c1454874ab183bce32e136ee is 50, key is test_row_0/C:col10/1733400920043/Put/seqid=0 2024-12-05T12:15:22,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742245_1421 (size=12301) 2024-12-05T12:15:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T12:15:23,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/ebb3ec44c1454874ab183bce32e136ee 2024-12-05T12:15:23,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/ba0cc2830f76435a9d71b0e1a173d214 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ba0cc2830f76435a9d71b0e1a173d214 2024-12-05T12:15:23,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ba0cc2830f76435a9d71b0e1a173d214, entries=150, sequenceid=261, filesize=30.5 K 2024-12-05T12:15:23,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/405058246a8245deac51e51a3c600bdd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/405058246a8245deac51e51a3c600bdd 2024-12-05T12:15:23,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/405058246a8245deac51e51a3c600bdd, entries=150, sequenceid=261, filesize=12.0 K 2024-12-05T12:15:23,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/ebb3ec44c1454874ab183bce32e136ee as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/ebb3ec44c1454874ab183bce32e136ee 2024-12-05T12:15:23,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/ebb3ec44c1454874ab183bce32e136ee, entries=150, sequenceid=261, filesize=12.0 K 2024-12-05T12:15:23,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for 8ac3794477a85e9bc205a48eed1522fa in 1669ms, sequenceid=261, compaction requested=true 2024-12-05T12:15:23,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ac3794477a85e9bc205a48eed1522fa: 2024-12-05T12:15:23,033 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:23,033 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:23,033 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. after waiting 0 ms 2024-12-05T12:15:23,033 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:23,033 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 8ac3794477a85e9bc205a48eed1522fa 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-05T12:15:23,033 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. because compaction request was cancelled 2024-12-05T12:15:23,033 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:A 2024-12-05T12:15:23,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:A, priority=-2147483648, current under compaction store size is 0 2024-12-05T12:15:23,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:23,033 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. because compaction request was cancelled 2024-12-05T12:15:23,033 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:B 2024-12-05T12:15:23,033 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=A 2024-12-05T12:15:23,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:B, priority=-2147483648, current under compaction store size is 0 2024-12-05T12:15:23,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:23,033 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:23,033 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. because compaction request was cancelled 2024-12-05T12:15:23,033 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=B 2024-12-05T12:15:23,033 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ac3794477a85e9bc205a48eed1522fa:C 2024-12-05T12:15:23,033 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:23,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ac3794477a85e9bc205a48eed1522fa:C, priority=-2147483648, current under compaction store size is 0 2024-12-05T12:15:23,033 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ac3794477a85e9bc205a48eed1522fa, store=C 2024-12-05T12:15:23,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:23,033 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:23,039 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e835cc67801b45f3bafa345ed00e49cf_8ac3794477a85e9bc205a48eed1522fa is 50, key is test_row_2/A:col10/1733400922203/Put/seqid=0 2024-12-05T12:15:23,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742246_1422 (size=7374) 2024-12-05T12:15:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T12:15:23,446 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:23,449 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e835cc67801b45f3bafa345ed00e49cf_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e835cc67801b45f3bafa345ed00e49cf_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:23,450 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/213415faba504af6bb225920ba562308, store: [table=TestAcidGuarantees family=A region=8ac3794477a85e9bc205a48eed1522fa] 2024-12-05T12:15:23,450 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/213415faba504af6bb225920ba562308 is 175, key is test_row_2/A:col10/1733400922203/Put/seqid=0 2024-12-05T12:15:23,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742247_1423 (size=13865) 2024-12-05T12:15:23,855 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/213415faba504af6bb225920ba562308 2024-12-05T12:15:23,868 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/54e23ce8d21f4a1a81fa442179e5cfb4 is 50, key is test_row_2/B:col10/1733400922203/Put/seqid=0 2024-12-05T12:15:23,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742248_1424 (size=7415) 2024-12-05T12:15:24,273 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/54e23ce8d21f4a1a81fa442179e5cfb4 2024-12-05T12:15:24,286 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/7cc3391629424d438cab9b9b9fc0bf09 is 50, key is test_row_2/C:col10/1733400922203/Put/seqid=0 2024-12-05T12:15:24,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742249_1425 (size=7415) 2024-12-05T12:15:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T12:15:24,692 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/7cc3391629424d438cab9b9b9fc0bf09 2024-12-05T12:15:24,699 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/A/213415faba504af6bb225920ba562308 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/213415faba504af6bb225920ba562308 2024-12-05T12:15:24,704 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/213415faba504af6bb225920ba562308, entries=50, sequenceid=268, filesize=13.5 K 2024-12-05T12:15:24,704 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/B/54e23ce8d21f4a1a81fa442179e5cfb4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/54e23ce8d21f4a1a81fa442179e5cfb4 2024-12-05T12:15:24,707 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/54e23ce8d21f4a1a81fa442179e5cfb4, entries=50, sequenceid=268, filesize=7.2 K 2024-12-05T12:15:24,707 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/.tmp/C/7cc3391629424d438cab9b9b9fc0bf09 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/7cc3391629424d438cab9b9b9fc0bf09 2024-12-05T12:15:24,710 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/7cc3391629424d438cab9b9b9fc0bf09, entries=50, sequenceid=268, filesize=7.2 K 2024-12-05T12:15:24,710 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8ac3794477a85e9bc205a48eed1522fa in 1677ms, sequenceid=268, compaction requested=true 2024-12-05T12:15:24,714 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/recovered.edits/271.seqid, newMaxSeqId=271, maxSeqId=4 2024-12-05T12:15:24,715 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. 2024-12-05T12:15:24,715 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 8ac3794477a85e9bc205a48eed1522fa: Waiting for close lock at 1733400922374Running coprocessor pre-close hooks at 1733400922374Disabling compacts and flushes for region at 1733400922374Disabling writes for close at 1733400923033 (+659 ms)Obtaining lock to block concurrent updates at 1733400923033Preparing flush snapshotting stores in 8ac3794477a85e9bc205a48eed1522fa at 1733400923033Finished memstore snapshotting TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733400923033Flushing stores of TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa. at 1733400923034 (+1 ms)Flushing 8ac3794477a85e9bc205a48eed1522fa/A: creating writer at 1733400923034Flushing mob file 8ac3794477a85e9bc205a48eed1522fa/A: appending metadata at 1733400923039 (+5 ms)Flushing mob file 8ac3794477a85e9bc205a48eed1522fa/A: closing flushed file at 1733400923039Flushing 8ac3794477a85e9bc205a48eed1522fa/A: appending metadata at 1733400923450 (+411 ms)Flushing 8ac3794477a85e9bc205a48eed1522fa/A: closing flushed file at 1733400923450Flushing 8ac3794477a85e9bc205a48eed1522fa/B: creating writer at 1733400923865 (+415 ms)Flushing 8ac3794477a85e9bc205a48eed1522fa/B: appending metadata at 1733400923868 (+3 ms)Flushing 8ac3794477a85e9bc205a48eed1522fa/B: closing flushed file at 1733400923868Flushing 8ac3794477a85e9bc205a48eed1522fa/C: creating writer at 1733400924283 (+415 ms)Flushing 8ac3794477a85e9bc205a48eed1522fa/C: appending metadata at 1733400924286 (+3 ms)Flushing 8ac3794477a85e9bc205a48eed1522fa/C: closing flushed file at 1733400924286Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fa1fb53: reopening flushed file at 1733400924698 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33bc8cbf: reopening flushed file at 1733400924704 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@501358b2: reopening flushed file at 1733400924707 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8ac3794477a85e9bc205a48eed1522fa in 1677ms, sequenceid=268, compaction requested=true at 1733400924710 (+3 ms)Writing region close event to WAL at 1733400924711 (+1 ms)Running coprocessor post-close hooks at 1733400924715 (+4 ms)Closed at 1733400924715 2024-12-05T12:15:24,717 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:24,717 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=8ac3794477a85e9bc205a48eed1522fa, regionState=CLOSED 2024-12-05T12:15:24,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:15:24,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-05T12:15:24,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 8ac3794477a85e9bc205a48eed1522fa, server=9ea9abad0e9b,44759,1733400796278 in 2.5010 sec 2024-12-05T12:15:24,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-05T12:15:24,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ac3794477a85e9bc205a48eed1522fa, UNASSIGN in 2.5070 sec 2024-12-05T12:15:24,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-05T12:15:24,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5120 sec 2024-12-05T12:15:24,730 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400924730"}]},"ts":"1733400924730"} 2024-12-05T12:15:24,731 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T12:15:24,731 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T12:15:24,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5240 sec 2024-12-05T12:15:25,895 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T12:15:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T12:15:26,363 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:26,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T12:15:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:26,367 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T12:15:26,368 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:26,371 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,375 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/recovered.edits] 2024-12-05T12:15:26,379 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/13bb44384edb4d428a01729b10b3173b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/13bb44384edb4d428a01729b10b3173b 2024-12-05T12:15:26,380 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/213415faba504af6bb225920ba562308 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/213415faba504af6bb225920ba562308 2024-12-05T12:15:26,381 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/341bdafcd57745dd9cab280afe8cc6f3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/341bdafcd57745dd9cab280afe8cc6f3 2024-12-05T12:15:26,382 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ba0cc2830f76435a9d71b0e1a173d214 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ba0cc2830f76435a9d71b0e1a173d214 2024-12-05T12:15:26,384 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ca53347e024a488fb9cec9b63ea8745a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/A/ca53347e024a488fb9cec9b63ea8745a 2024-12-05T12:15:26,386 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/405058246a8245deac51e51a3c600bdd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/405058246a8245deac51e51a3c600bdd 2024-12-05T12:15:26,387 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/54e23ce8d21f4a1a81fa442179e5cfb4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/54e23ce8d21f4a1a81fa442179e5cfb4 2024-12-05T12:15:26,388 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/614a3fcf4d334b9e9fb249d6e17c8683 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/614a3fcf4d334b9e9fb249d6e17c8683 2024-12-05T12:15:26,388 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/7ef74a524f9247fbb5f17fa70034856f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/7ef74a524f9247fbb5f17fa70034856f 2024-12-05T12:15:26,389 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/b909d8352ec24b1289543cd27d8e6392 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/B/b909d8352ec24b1289543cd27d8e6392 2024-12-05T12:15:26,392 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/55ce2cb10352470282b3d0974bb419b8 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/55ce2cb10352470282b3d0974bb419b8 2024-12-05T12:15:26,393 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/636229e1e4b849bd88cd1af7b137f591 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/636229e1e4b849bd88cd1af7b137f591 2024-12-05T12:15:26,395 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/7cc3391629424d438cab9b9b9fc0bf09 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/7cc3391629424d438cab9b9b9fc0bf09 2024-12-05T12:15:26,396 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/9452aa6b7d2f4e63a89d76421269d57c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/9452aa6b7d2f4e63a89d76421269d57c 2024-12-05T12:15:26,397 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/ebb3ec44c1454874ab183bce32e136ee to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/C/ebb3ec44c1454874ab183bce32e136ee 2024-12-05T12:15:26,400 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/recovered.edits/271.seqid to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa/recovered.edits/271.seqid 2024-12-05T12:15:26,400 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,400 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T12:15:26,401 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T12:15:26,402 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T12:15:26,405 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120506e5282357664f59aefa4eea271c799b_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120506e5282357664f59aefa4eea271c799b_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,406 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050ed37c524e0e4650a01f784c19424797_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050ed37c524e0e4650a01f784c19424797_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,407 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051a8b4328103841739b976060cc68c520_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051a8b4328103841739b976060cc68c520_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,409 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051b9ea33f6370460fab1dd7078b649a87_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051b9ea33f6370460fab1dd7078b649a87_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,410 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120521abf84cd4064d90b066bf7c0214ce7a_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120521abf84cd4064d90b066bf7c0214ce7a_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,411 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120529cb0d88b4144361a2dbd9de3c8f66d6_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120529cb0d88b4144361a2dbd9de3c8f66d6_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,412 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120542b8edbfda2a476c97356ae22f1c38a3_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120542b8edbfda2a476c97356ae22f1c38a3_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,413 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120544622073c19b4396a50f2f5eefa970a6_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120544622073c19b4396a50f2f5eefa970a6_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,414 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120555533270071f453c967ad5771485eacd_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120555533270071f453c967ad5771485eacd_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,415 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205641cf6a3355d4415b6fe039f093d90c8_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205641cf6a3355d4415b6fe039f093d90c8_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,416 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120583f22d8818bb4cdaac01d519ac582e21_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120583f22d8818bb4cdaac01d519ac582e21_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,417 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120584af62d8d6f74e61bf347f57c3b5d4db_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120584af62d8d6f74e61bf347f57c3b5d4db_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,417 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ab0e72df50f547a78d56cc94ca802c4e_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ab0e72df50f547a78d56cc94ca802c4e_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,418 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205acbe386c8d3044aaa7834eee6dc4495f_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205acbe386c8d3044aaa7834eee6dc4495f_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,419 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cbc0bc574acd4f85baa31bc785c10ccb_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cbc0bc574acd4f85baa31bc785c10ccb_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,420 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e835cc67801b45f3bafa345ed00e49cf_8ac3794477a85e9bc205a48eed1522fa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e835cc67801b45f3bafa345ed00e49cf_8ac3794477a85e9bc205a48eed1522fa 2024-12-05T12:15:26,420 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T12:15:26,422 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:26,424 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T12:15:26,425 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T12:15:26,426 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:26,426 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T12:15:26,426 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733400926426"}]},"ts":"9223372036854775807"} 2024-12-05T12:15:26,428 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T12:15:26,428 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8ac3794477a85e9bc205a48eed1522fa, NAME => 'TestAcidGuarantees,,1733400897006.8ac3794477a85e9bc205a48eed1522fa.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T12:15:26,428 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T12:15:26,428 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733400926428"}]},"ts":"9223372036854775807"} 2024-12-05T12:15:26,430 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T12:15:26,430 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:26,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-12-05T12:15:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T12:15:26,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T12:15:26,474 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:26,488 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-144804023_22 at /127.0.0.1:36510 [Waiting for operation #506] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-660316091_22 at /127.0.0.1:36514 [Waiting for operation #496] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-144804023_22 at /127.0.0.1:33924 [Waiting for operation #606] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-660316091_22 at /127.0.0.1:36516 [Waiting for operation #507] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/9ea9abad0e9b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=572 (was 638), ProcessCount=11 (was 11), AvailableMemoryMB=3912 (was 3610) - AvailableMemoryMB LEAK? - 2024-12-05T12:15:26,498 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=572, ProcessCount=11, AvailableMemoryMB=3912 2024-12-05T12:15:26,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T12:15:26,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:15:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:26,501 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T12:15:26,501 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:26,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-12-05T12:15:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T12:15:26,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T12:15:26,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742250_1426 (size=1061) 2024-12-05T12:15:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T12:15:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T12:15:26,912 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e 2024-12-05T12:15:26,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742251_1427 (size=53) 2024-12-05T12:15:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T12:15:27,323 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:15:27,323 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7a7e5a7bd5645c444a2a620ce27fbe6a, disabling compactions & flushes 2024-12-05T12:15:27,323 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:27,323 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:27,323 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. after waiting 0 ms 2024-12-05T12:15:27,323 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:27,323 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:27,323 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: Waiting for close lock at 1733400927323Disabling compacts and flushes for region at 1733400927323Disabling writes for close at 1733400927323Writing region close event to WAL at 1733400927323Closed at 1733400927323 2024-12-05T12:15:27,324 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T12:15:27,325 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733400927324"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733400927324"}]},"ts":"1733400927324"} 2024-12-05T12:15:27,327 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T12:15:27,329 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T12:15:27,329 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400927329"}]},"ts":"1733400927329"} 2024-12-05T12:15:27,331 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T12:15:27,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a7e5a7bd5645c444a2a620ce27fbe6a, ASSIGN}] 2024-12-05T12:15:27,334 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a7e5a7bd5645c444a2a620ce27fbe6a, ASSIGN 2024-12-05T12:15:27,335 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a7e5a7bd5645c444a2a620ce27fbe6a, ASSIGN; state=OFFLINE, location=9ea9abad0e9b,44759,1733400796278; forceNewPlan=false, retain=false 2024-12-05T12:15:27,486 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=7a7e5a7bd5645c444a2a620ce27fbe6a, regionState=OPENING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:27,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a7e5a7bd5645c444a2a620ce27fbe6a, ASSIGN because future has completed 2024-12-05T12:15:27,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:15:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T12:15:27,645 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:27,645 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:15:27,647 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,647 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:15:27,647 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,647 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,649 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,651 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:27,651 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a7e5a7bd5645c444a2a620ce27fbe6a columnFamilyName A 2024-12-05T12:15:27,651 DEBUG [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:27,652 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] regionserver.HStore(327): Store=7a7e5a7bd5645c444a2a620ce27fbe6a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:27,652 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,653 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:27,653 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a7e5a7bd5645c444a2a620ce27fbe6a columnFamilyName B 2024-12-05T12:15:27,653 DEBUG [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:27,654 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] regionserver.HStore(327): Store=7a7e5a7bd5645c444a2a620ce27fbe6a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:27,654 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,655 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:27,655 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a7e5a7bd5645c444a2a620ce27fbe6a columnFamilyName C 2024-12-05T12:15:27,655 DEBUG [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:27,656 INFO [StoreOpener-7a7e5a7bd5645c444a2a620ce27fbe6a-1 {}] regionserver.HStore(327): Store=7a7e5a7bd5645c444a2a620ce27fbe6a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:27,656 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:27,656 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,657 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,657 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,658 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,658 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,659 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:15:27,661 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,663 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:15:27,663 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 7a7e5a7bd5645c444a2a620ce27fbe6a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72473142, jitterRate=0.07993397116661072}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:15:27,663 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:27,664 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: Running coprocessor pre-open hook at 1733400927648Writing region info on filesystem at 1733400927648Initializing all the Stores at 1733400927649 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400927649Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400927649Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400927649Cleaning up temporary data from old regions at 1733400927658 (+9 ms)Running coprocessor post-open hooks at 1733400927663 (+5 ms)Region opened successfully at 1733400927664 (+1 ms) 2024-12-05T12:15:27,665 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., pid=118, masterSystemTime=1733400927639 2024-12-05T12:15:27,667 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:27,667 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:27,667 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=7a7e5a7bd5645c444a2a620ce27fbe6a, regionState=OPEN, openSeqNum=2, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:27,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:15:27,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-05T12:15:27,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 in 183 msec 2024-12-05T12:15:27,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-12-05T12:15:27,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a7e5a7bd5645c444a2a620ce27fbe6a, ASSIGN in 340 msec 2024-12-05T12:15:27,675 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T12:15:27,675 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400927675"}]},"ts":"1733400927675"} 2024-12-05T12:15:27,676 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T12:15:27,677 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T12:15:27,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-12-05T12:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T12:15:28,654 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:28,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca175cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,659 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1289e135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,661 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e62b3a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,662 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,670 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T12:15:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ea9abad0e9b,34103,1733400795471 2024-12-05T12:15:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6022c4af 2024-12-05T12:15:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T12:15:28,671 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T12:15:28,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491481f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,673 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b4123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,675 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46973ca9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7631696d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,676 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ee7ab0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,677 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfeb066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126b8236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,679 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fef3474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,680 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c1d9dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea00808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,682 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cee2c8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,683 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10172100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1589e05f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,685 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221af0bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,686 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@682622f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c19380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,688 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c30f082, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,689 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3a4166, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b23f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,690 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d62675c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261e480b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7d791a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,693 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb4ca4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,694 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ca8e24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b3a94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,696 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5241a5e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,697 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36de1239, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c68a1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:28,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:28,699 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217f41b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:28,700 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:28,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f6b3cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:28,701 DEBUG [Thread-1933 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,701 DEBUG [Thread-1931 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,701 DEBUG [Thread-1942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,701 DEBUG [Thread-1935 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,701 DEBUG [Thread-1940 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:28,701 DEBUG [Thread-1946 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,702 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,702 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,703 DEBUG [Thread-1944 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,703 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,703 DEBUG [Thread-1938 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-05T12:15:28,703 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,703 DEBUG [Thread-1938 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T12:15:28,704 DEBUG [Thread-1929 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:15:28,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:15:28,706 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:15:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:15:28,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:28,708 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:15:28,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:15:28,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:15:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:15:28,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:15:28,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2] 2024-12-05T12:15:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:28,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:15:28,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:28,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:28,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:28,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:28,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:28,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:28,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/29c8433b022243e99086d7caf925804f is 50, key is test_row_0/A:col10/1733400928712/Put/seqid=0 2024-12-05T12:15:28,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400988738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:28,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733400928722, max timestamp=1733400928748Num uniques 859; 2024-12-05T12:15:28,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733400928722, max timestamp=1733400928748Num uniques 859; 2024-12-05T12:15:28,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:15:28,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:15:28,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733400928722, max timestamp=1733400928748Num uniques 859; 2024-12-05T12:15:28,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:15:28,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742252_1428 (size=12001) 2024-12-05T12:15:28,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/29c8433b022243e99086d7caf925804f 2024-12-05T12:15:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400988777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733400988778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/20190dcc1f48483088d58448c83da3b3 is 50, key is test_row_0/B:col10/1733400928712/Put/seqid=0 2024-12-05T12:15:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:28,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742253_1429 (size=12001) 2024-12-05T12:15:28,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/20190dcc1f48483088d58448c83da3b3 2024-12-05T12:15:28,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733400988784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:28,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733400988784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:28,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/fc35426e63f147049d5ba2aafaa22966 is 50, key is test_row_0/C:col10/1733400928712/Put/seqid=0 2024-12-05T12:15:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T12:15:28,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742254_1430 (size=12001) 2024-12-05T12:15:28,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T12:15:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:28,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:28,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400988852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:28,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400988893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733400988893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:28,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733400988902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:28,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733400988902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T12:15:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:29,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T12:15:29,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400989073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400989102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733400989102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733400989112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733400989112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T12:15:29,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:29,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:29,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:29,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:29,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:29,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/fc35426e63f147049d5ba2aafaa22966 2024-12-05T12:15:29,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/29c8433b022243e99086d7caf925804f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/29c8433b022243e99086d7caf925804f 2024-12-05T12:15:29,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/29c8433b022243e99086d7caf925804f, entries=150, sequenceid=12, filesize=11.7 K 2024-12-05T12:15:29,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/20190dcc1f48483088d58448c83da3b3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/20190dcc1f48483088d58448c83da3b3 2024-12-05T12:15:29,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/20190dcc1f48483088d58448c83da3b3, entries=150, sequenceid=12, filesize=11.7 K 2024-12-05T12:15:29,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/fc35426e63f147049d5ba2aafaa22966 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/fc35426e63f147049d5ba2aafaa22966 2024-12-05T12:15:29,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/fc35426e63f147049d5ba2aafaa22966, entries=150, sequenceid=12, filesize=11.7 K 2024-12-05T12:15:29,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 518ms, sequenceid=12, compaction requested=false 2024-12-05T12:15:29,232 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T12:15:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:29,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T12:15:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:29,321 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-05T12:15:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/f4f07f4afb424e11a4a9ffea61f65b36 is 50, key is test_row_0/A:col10/1733400928782/Put/seqid=0 2024-12-05T12:15:29,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742255_1431 (size=12001) 2024-12-05T12:15:29,330 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/f4f07f4afb424e11a4a9ffea61f65b36 2024-12-05T12:15:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T12:15:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/783040155d704e5f81cbbfc02b9d40d7 is 50, key is test_row_0/B:col10/1733400928782/Put/seqid=0 2024-12-05T12:15:29,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742256_1432 (size=12001) 2024-12-05T12:15:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:29,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:29,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733400989413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400989417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400989420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733400989422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733400989422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400989532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400989533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,741 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/783040155d704e5f81cbbfc02b9d40d7 2024-12-05T12:15:29,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400989742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400989742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/7b5088f9b4bb446c982db20ebdd16e8f is 50, key is test_row_0/C:col10/1733400928782/Put/seqid=0 2024-12-05T12:15:29,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742257_1433 (size=12001) 2024-12-05T12:15:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T12:15:29,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733400989923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733400989932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733400989932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400990052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400990052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,151 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/7b5088f9b4bb446c982db20ebdd16e8f 2024-12-05T12:15:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/f4f07f4afb424e11a4a9ffea61f65b36 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/f4f07f4afb424e11a4a9ffea61f65b36 2024-12-05T12:15:30,158 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/f4f07f4afb424e11a4a9ffea61f65b36, entries=150, sequenceid=46, filesize=11.7 K 2024-12-05T12:15:30,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/783040155d704e5f81cbbfc02b9d40d7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/783040155d704e5f81cbbfc02b9d40d7 2024-12-05T12:15:30,161 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/783040155d704e5f81cbbfc02b9d40d7, entries=150, sequenceid=46, filesize=11.7 K 2024-12-05T12:15:30,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/7b5088f9b4bb446c982db20ebdd16e8f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7b5088f9b4bb446c982db20ebdd16e8f 2024-12-05T12:15:30,165 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7b5088f9b4bb446c982db20ebdd16e8f, entries=150, sequenceid=46, filesize=11.7 K 2024-12-05T12:15:30,166 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 845ms, sequenceid=46, compaction requested=false 2024-12-05T12:15:30,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:30,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:30,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-05T12:15:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-05T12:15:30,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-05T12:15:30,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-12-05T12:15:30,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4690 sec 2024-12-05T12:15:30,233 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T12:15:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:30,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T12:15:30,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:30,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:30,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:30,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:30,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:30,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:30,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/eccbd4bc54824cfea03ebbf4ac67374e is 50, key is test_row_0/A:col10/1733400929419/Put/seqid=0 2024-12-05T12:15:30,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742258_1434 (size=14341) 2024-12-05T12:15:30,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400990614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400990615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400990723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400990733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T12:15:30,853 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:30,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-05T12:15:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T12:15:30,855 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:30,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:30,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:30,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400990932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733400990943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733400990943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400990943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,948 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,948 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733400990952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:30,957 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T12:15:30,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/eccbd4bc54824cfea03ebbf4ac67374e 2024-12-05T12:15:30,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/7a10dbfd7a9d4523be8deaa8b773151c is 50, key is test_row_0/B:col10/1733400929419/Put/seqid=0 2024-12-05T12:15:30,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742259_1435 (size=12001) 2024-12-05T12:15:31,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T12:15:31,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T12:15:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T12:15:31,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400991243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:31,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400991253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:31,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T12:15:31,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:31,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/7a10dbfd7a9d4523be8deaa8b773151c 2024-12-05T12:15:31,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/ffd2520d638d4ee7a9527365638276b5 is 50, key is test_row_0/C:col10/1733400929419/Put/seqid=0 2024-12-05T12:15:31,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742260_1436 (size=12001) 2024-12-05T12:15:31,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T12:15:31,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:31,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T12:15:31,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T12:15:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400991752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:31,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T12:15:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:31,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400991773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:31,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/ffd2520d638d4ee7a9527365638276b5 2024-12-05T12:15:31,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/eccbd4bc54824cfea03ebbf4ac67374e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/eccbd4bc54824cfea03ebbf4ac67374e 2024-12-05T12:15:31,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/eccbd4bc54824cfea03ebbf4ac67374e, entries=200, sequenceid=58, filesize=14.0 K 2024-12-05T12:15:31,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/7a10dbfd7a9d4523be8deaa8b773151c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/7a10dbfd7a9d4523be8deaa8b773151c 2024-12-05T12:15:31,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/7a10dbfd7a9d4523be8deaa8b773151c, entries=150, sequenceid=58, filesize=11.7 K 2024-12-05T12:15:31,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/ffd2520d638d4ee7a9527365638276b5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/ffd2520d638d4ee7a9527365638276b5 2024-12-05T12:15:31,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/ffd2520d638d4ee7a9527365638276b5, entries=150, sequenceid=58, filesize=11.7 K 2024-12-05T12:15:31,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 1268ms, sequenceid=58, compaction requested=true 2024-12-05T12:15:31,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:31,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:31,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:31,833 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:31,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:31,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:31,833 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:31,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:31,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:31,836 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:31,836 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:31,836 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/B is initiating minor compaction (all files) 2024-12-05T12:15:31,836 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/A is initiating minor compaction (all files) 2024-12-05T12:15:31,836 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/B in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,836 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/A in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,836 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/20190dcc1f48483088d58448c83da3b3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/783040155d704e5f81cbbfc02b9d40d7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/7a10dbfd7a9d4523be8deaa8b773151c] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=35.2 K 2024-12-05T12:15:31,836 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/29c8433b022243e99086d7caf925804f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/f4f07f4afb424e11a4a9ffea61f65b36, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/eccbd4bc54824cfea03ebbf4ac67374e] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=37.4 K 2024-12-05T12:15:31,836 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 20190dcc1f48483088d58448c83da3b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733400928712 2024-12-05T12:15:31,836 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29c8433b022243e99086d7caf925804f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733400928712 2024-12-05T12:15:31,836 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f07f4afb424e11a4a9ffea61f65b36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733400928774 2024-12-05T12:15:31,836 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 783040155d704e5f81cbbfc02b9d40d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733400928774 2024-12-05T12:15:31,837 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting eccbd4bc54824cfea03ebbf4ac67374e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733400929407 2024-12-05T12:15:31,837 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a10dbfd7a9d4523be8deaa8b773151c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733400929407 2024-12-05T12:15:31,852 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:31,853 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#A#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:31,853 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/33dd5f0c65c849b2873537a91d0abf84 is 50, key is test_row_0/B:col10/1733400929419/Put/seqid=0 2024-12-05T12:15:31,853 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/11beab3532e842d396a7a21531cc35a4 is 50, key is test_row_0/A:col10/1733400929419/Put/seqid=0 2024-12-05T12:15:31,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742261_1437 (size=12104) 2024-12-05T12:15:31,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742262_1438 (size=12104) 2024-12-05T12:15:31,866 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/11beab3532e842d396a7a21531cc35a4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/11beab3532e842d396a7a21531cc35a4 2024-12-05T12:15:31,869 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/A of 7a7e5a7bd5645c444a2a620ce27fbe6a into 11beab3532e842d396a7a21531cc35a4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:31,869 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:31,869 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/A, priority=13, startTime=1733400931833; duration=0sec 2024-12-05T12:15:31,869 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:31,870 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:A 2024-12-05T12:15:31,870 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:31,870 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:31,870 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/C is initiating minor compaction (all files) 2024-12-05T12:15:31,870 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/C in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,870 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/fc35426e63f147049d5ba2aafaa22966, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7b5088f9b4bb446c982db20ebdd16e8f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/ffd2520d638d4ee7a9527365638276b5] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=35.2 K 2024-12-05T12:15:31,871 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc35426e63f147049d5ba2aafaa22966, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733400928712 2024-12-05T12:15:31,871 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5088f9b4bb446c982db20ebdd16e8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733400928774 2024-12-05T12:15:31,871 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd2520d638d4ee7a9527365638276b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733400929407 2024-12-05T12:15:31,876 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:31,876 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/76720493133543e8a759c61ead1e76b3 is 50, key is test_row_0/C:col10/1733400929419/Put/seqid=0 2024-12-05T12:15:31,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742263_1439 (size=12104) 2024-12-05T12:15:31,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T12:15:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:31,924 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T12:15:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:31,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/b2bca433899843b58bda5dae44b32e0b is 50, key is test_row_0/A:col10/1733400930612/Put/seqid=0 2024-12-05T12:15:31,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742264_1440 (size=12001) 2024-12-05T12:15:31,947 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/b2bca433899843b58bda5dae44b32e0b 2024-12-05T12:15:31,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/773b4dfafb96493ab0c96166f21c79b7 is 50, key is test_row_0/B:col10/1733400930612/Put/seqid=0 2024-12-05T12:15:31,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742265_1441 (size=12001) 2024-12-05T12:15:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T12:15:32,264 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/33dd5f0c65c849b2873537a91d0abf84 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/33dd5f0c65c849b2873537a91d0abf84 2024-12-05T12:15:32,267 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/B of 7a7e5a7bd5645c444a2a620ce27fbe6a into 33dd5f0c65c849b2873537a91d0abf84(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:32,267 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:32,267 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/B, priority=13, startTime=1733400931833; duration=0sec 2024-12-05T12:15:32,267 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:32,267 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:B 2024-12-05T12:15:32,284 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/76720493133543e8a759c61ead1e76b3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/76720493133543e8a759c61ead1e76b3 2024-12-05T12:15:32,287 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/C of 7a7e5a7bd5645c444a2a620ce27fbe6a into 76720493133543e8a759c61ead1e76b3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:32,287 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:32,287 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/C, priority=13, startTime=1733400931833; duration=0sec 2024-12-05T12:15:32,287 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:32,287 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:C 2024-12-05T12:15:32,359 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/773b4dfafb96493ab0c96166f21c79b7 2024-12-05T12:15:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/b1fa00e4b4b04ba9ba462287537f5283 is 50, key is test_row_0/C:col10/1733400930612/Put/seqid=0 2024-12-05T12:15:32,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742266_1442 (size=12001) 2024-12-05T12:15:32,769 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/b1fa00e4b4b04ba9ba462287537f5283 2024-12-05T12:15:32,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/b2bca433899843b58bda5dae44b32e0b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/b2bca433899843b58bda5dae44b32e0b 2024-12-05T12:15:32,775 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/b2bca433899843b58bda5dae44b32e0b, entries=150, sequenceid=79, filesize=11.7 K 2024-12-05T12:15:32,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/773b4dfafb96493ab0c96166f21c79b7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/773b4dfafb96493ab0c96166f21c79b7 2024-12-05T12:15:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:32,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:32,778 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/773b4dfafb96493ab0c96166f21c79b7, entries=150, sequenceid=79, filesize=11.7 K 2024-12-05T12:15:32,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/b1fa00e4b4b04ba9ba462287537f5283 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b1fa00e4b4b04ba9ba462287537f5283 2024-12-05T12:15:32,782 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b1fa00e4b4b04ba9ba462287537f5283, entries=150, sequenceid=79, filesize=11.7 K 2024-12-05T12:15:32,783 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 859ms, sequenceid=79, compaction requested=false 2024-12-05T12:15:32,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:32,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:32,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-05T12:15:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-05T12:15:32,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-05T12:15:32,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9290 sec 2024-12-05T12:15:32,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.9340 sec 2024-12-05T12:15:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:32,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:15:32,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:32,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:32,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:32,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:32,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:32,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:32,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/d43d6ed3ef1c43699219edd07eeefbae is 50, key is test_row_0/A:col10/1733400932784/Put/seqid=0 2024-12-05T12:15:32,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742267_1443 (size=11997) 2024-12-05T12:15:32,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/d43d6ed3ef1c43699219edd07eeefbae 2024-12-05T12:15:32,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/9fd2ae093434407ea425ea781ba2c9e1 is 50, key is test_row_0/B:col10/1733400932784/Put/seqid=0 2024-12-05T12:15:32,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742268_1444 (size=9657) 2024-12-05T12:15:32,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400932833, max timestamp=1733400932850Num uniques 859; 2024-12-05T12:15:32,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:32,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400932833, max timestamp=1733400932850Num uniques 859; 2024-12-05T12:15:32,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400932833, max timestamp=1733400932850Num uniques 859; 2024-12-05T12:15:32,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:32,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:32,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400992903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:32,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400992911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:32,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733400992963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:32,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733400992963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:32,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733400992963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:32,967 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:32,968 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:32,968 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T12:15:33,003 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:33,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-05T12:15:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T12:15:33,004 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:33,005 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:33,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:33,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400993013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:33,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400993023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T12:15:33,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T12:15:33,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/9fd2ae093434407ea425ea781ba2c9e1 2024-12-05T12:15:33,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/721b225d62a747b69d3db70e5d26d2cb is 50, key is test_row_0/C:col10/1733400932784/Put/seqid=0 2024-12-05T12:15:33,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742269_1445 (size=9657) 2024-12-05T12:15:33,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400993223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:33,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400993233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:33,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T12:15:33,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:33,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T12:15:33,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T12:15:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400993532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400993543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:33,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T12:15:33,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:33,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:33,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/721b225d62a747b69d3db70e5d26d2cb 2024-12-05T12:15:33,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/d43d6ed3ef1c43699219edd07eeefbae as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/d43d6ed3ef1c43699219edd07eeefbae 2024-12-05T12:15:33,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/d43d6ed3ef1c43699219edd07eeefbae, entries=150, sequenceid=91, filesize=11.7 K 2024-12-05T12:15:33,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/9fd2ae093434407ea425ea781ba2c9e1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9fd2ae093434407ea425ea781ba2c9e1 2024-12-05T12:15:33,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9fd2ae093434407ea425ea781ba2c9e1, entries=100, sequenceid=91, filesize=9.4 K 2024-12-05T12:15:33,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/721b225d62a747b69d3db70e5d26d2cb as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/721b225d62a747b69d3db70e5d26d2cb 2024-12-05T12:15:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T12:15:33,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/721b225d62a747b69d3db70e5d26d2cb, entries=100, sequenceid=91, filesize=9.4 K 2024-12-05T12:15:33,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 845ms, sequenceid=91, compaction requested=true 2024-12-05T12:15:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:33,635 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:33,635 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:33,636 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:33,636 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:33,636 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/B is initiating minor compaction (all files) 2024-12-05T12:15:33,636 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/A is initiating minor compaction (all files) 2024-12-05T12:15:33,636 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/B in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,636 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/A in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,636 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/33dd5f0c65c849b2873537a91d0abf84, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/773b4dfafb96493ab0c96166f21c79b7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9fd2ae093434407ea425ea781ba2c9e1] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=33.0 K 2024-12-05T12:15:33,636 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/11beab3532e842d396a7a21531cc35a4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/b2bca433899843b58bda5dae44b32e0b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/d43d6ed3ef1c43699219edd07eeefbae] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=35.3 K 2024-12-05T12:15:33,636 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 33dd5f0c65c849b2873537a91d0abf84, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733400929407 2024-12-05T12:15:33,636 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11beab3532e842d396a7a21531cc35a4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733400929407 2024-12-05T12:15:33,636 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 773b4dfafb96493ab0c96166f21c79b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733400930608 2024-12-05T12:15:33,636 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2bca433899843b58bda5dae44b32e0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733400930608 2024-12-05T12:15:33,636 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fd2ae093434407ea425ea781ba2c9e1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733400932784 2024-12-05T12:15:33,636 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting d43d6ed3ef1c43699219edd07eeefbae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733400932784 2024-12-05T12:15:33,642 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#A#compaction#372 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:33,642 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/c2ba0d4fef7b4c869e25f47d3dd133c2 is 50, key is test_row_0/A:col10/1733400932784/Put/seqid=0 2024-12-05T12:15:33,643 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#B#compaction#373 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:33,643 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/01c3b9b3f30e4fb3bc409675e26f0353 is 50, key is test_row_0/B:col10/1733400932784/Put/seqid=0 2024-12-05T12:15:33,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742271_1447 (size=12207) 2024-12-05T12:15:33,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742270_1446 (size=12207) 2024-12-05T12:15:33,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T12:15:33,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:33,767 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T12:15:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:33,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/163705d33a8941fc906730d55006ba6d is 50, key is test_row_0/A:col10/1733400932872/Put/seqid=0 2024-12-05T12:15:33,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742272_1448 (size=12001) 2024-12-05T12:15:34,050 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/01c3b9b3f30e4fb3bc409675e26f0353 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/01c3b9b3f30e4fb3bc409675e26f0353 2024-12-05T12:15:34,054 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/c2ba0d4fef7b4c869e25f47d3dd133c2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/c2ba0d4fef7b4c869e25f47d3dd133c2 2024-12-05T12:15:34,054 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/B of 7a7e5a7bd5645c444a2a620ce27fbe6a into 01c3b9b3f30e4fb3bc409675e26f0353(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:34,054 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:34,054 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/B, priority=13, startTime=1733400933635; duration=0sec 2024-12-05T12:15:34,054 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:34,054 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:B 2024-12-05T12:15:34,054 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:34,055 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:34,055 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/C is initiating minor compaction (all files) 2024-12-05T12:15:34,055 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/C in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:34,055 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/76720493133543e8a759c61ead1e76b3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b1fa00e4b4b04ba9ba462287537f5283, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/721b225d62a747b69d3db70e5d26d2cb] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=33.0 K 2024-12-05T12:15:34,055 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 76720493133543e8a759c61ead1e76b3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733400929407 2024-12-05T12:15:34,056 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b1fa00e4b4b04ba9ba462287537f5283, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733400930608 2024-12-05T12:15:34,056 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 721b225d62a747b69d3db70e5d26d2cb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733400932784 2024-12-05T12:15:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:34,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:34,059 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/A of 7a7e5a7bd5645c444a2a620ce27fbe6a into c2ba0d4fef7b4c869e25f47d3dd133c2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:34,059 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:34,059 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/A, priority=13, startTime=1733400933635; duration=0sec 2024-12-05T12:15:34,059 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:34,059 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:A 2024-12-05T12:15:34,061 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:34,061 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/7123cde97a204829a1906777462cd10a is 50, key is test_row_0/C:col10/1733400932784/Put/seqid=0 2024-12-05T12:15:34,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742273_1449 (size=12207) 2024-12-05T12:15:34,070 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/7123cde97a204829a1906777462cd10a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7123cde97a204829a1906777462cd10a 2024-12-05T12:15:34,074 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/C of 7a7e5a7bd5645c444a2a620ce27fbe6a into 7123cde97a204829a1906777462cd10a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:34,074 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:34,074 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/C, priority=13, startTime=1733400933635; duration=0sec 2024-12-05T12:15:34,074 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:34,074 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:C 2024-12-05T12:15:34,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400994078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:34,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400994081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T12:15:34,175 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/163705d33a8941fc906730d55006ba6d 2024-12-05T12:15:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/a2cdf3a4de0245398c9a179fd86c0c63 is 50, key is test_row_0/B:col10/1733400932872/Put/seqid=0 2024-12-05T12:15:34,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742274_1450 (size=12001) 2024-12-05T12:15:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400994193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400994193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:34,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400994403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:34,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400994403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:34,585 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/a2cdf3a4de0245398c9a179fd86c0c63 2024-12-05T12:15:34,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/5a13006acd3f4931a5cc33b56be5c5b1 is 50, key is test_row_0/C:col10/1733400932872/Put/seqid=0 2024-12-05T12:15:34,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742275_1451 (size=12001) 2024-12-05T12:15:34,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400994713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:34,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400994713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:34,995 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/5a13006acd3f4931a5cc33b56be5c5b1 2024-12-05T12:15:34,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/163705d33a8941fc906730d55006ba6d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/163705d33a8941fc906730d55006ba6d 2024-12-05T12:15:35,001 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/163705d33a8941fc906730d55006ba6d, entries=150, sequenceid=128, filesize=11.7 K 2024-12-05T12:15:35,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/a2cdf3a4de0245398c9a179fd86c0c63 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/a2cdf3a4de0245398c9a179fd86c0c63 2024-12-05T12:15:35,003 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/a2cdf3a4de0245398c9a179fd86c0c63, entries=150, sequenceid=128, filesize=11.7 K 2024-12-05T12:15:35,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/5a13006acd3f4931a5cc33b56be5c5b1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5a13006acd3f4931a5cc33b56be5c5b1 2024-12-05T12:15:35,006 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5a13006acd3f4931a5cc33b56be5c5b1, entries=150, sequenceid=128, filesize=11.7 K 2024-12-05T12:15:35,007 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 1240ms, sequenceid=128, compaction requested=false 2024-12-05T12:15:35,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:35,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-05T12:15:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-05T12:15:35,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-05T12:15:35,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0030 sec 2024-12-05T12:15:35,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.0070 sec 2024-12-05T12:15:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T12:15:35,163 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:35,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-05T12:15:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T12:15:35,165 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:35,165 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:35,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:35,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T12:15:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:35,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/071fd700dec8474d93f1bda26d0ab6f0 is 50, key is test_row_0/A:col10/1733400934056/Put/seqid=0 2024-12-05T12:15:35,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742276_1452 (size=14541) 2024-12-05T12:15:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T12:15:35,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400935258, max timestamp=1733400935290Num uniques 859; 2024-12-05T12:15:35,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400935258, max timestamp=1733400935290Num uniques 859; 2024-12-05T12:15:35,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400935258, max timestamp=1733400935290Num uniques 859; 2024-12-05T12:15:35,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:15:35,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:15:35,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:15:35,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400995336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:35,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400995337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:35,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400995443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:35,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400995452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:35,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:35,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:35,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T12:15:35,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:35,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:35,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/071fd700dec8474d93f1bda26d0ab6f0 2024-12-05T12:15:35,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/c092ec43c55f4a8d98eb0fa39d0970c2 is 50, key is test_row_0/B:col10/1733400934056/Put/seqid=0 2024-12-05T12:15:35,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742277_1453 (size=12151) 2024-12-05T12:15:35,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400995653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:35,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400995663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:35,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T12:15:35,895 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T12:15:35,895 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T12:15:35,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:35,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:35,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400995962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:35,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400995973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:36,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/c092ec43c55f4a8d98eb0fa39d0970c2 2024-12-05T12:15:36,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/c612f36abfbb417c911672c573409eca is 50, key is test_row_0/C:col10/1733400934056/Put/seqid=0 2024-12-05T12:15:36,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742278_1454 (size=12151) 2024-12-05T12:15:36,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:36,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T12:15:36,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/c612f36abfbb417c911672c573409eca 2024-12-05T12:15:36,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/071fd700dec8474d93f1bda26d0ab6f0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/071fd700dec8474d93f1bda26d0ab6f0 2024-12-05T12:15:36,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/071fd700dec8474d93f1bda26d0ab6f0, entries=200, sequenceid=142, filesize=14.2 K 2024-12-05T12:15:36,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/c092ec43c55f4a8d98eb0fa39d0970c2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/c092ec43c55f4a8d98eb0fa39d0970c2 2024-12-05T12:15:36,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/c092ec43c55f4a8d98eb0fa39d0970c2, entries=150, sequenceid=142, filesize=11.9 K 2024-12-05T12:15:36,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/c612f36abfbb417c911672c573409eca as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/c612f36abfbb417c911672c573409eca 2024-12-05T12:15:36,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/c612f36abfbb417c911672c573409eca, entries=150, sequenceid=142, filesize=11.9 K 2024-12-05T12:15:36,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 1241ms, sequenceid=142, compaction requested=true 2024-12-05T12:15:36,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:36,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:36,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:36,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:36,467 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:36,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:36,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:36,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:36,467 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:36,468 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:36,468 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:36,468 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/B is initiating minor compaction (all files) 2024-12-05T12:15:36,468 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/A is initiating minor compaction (all files) 2024-12-05T12:15:36,468 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/B in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,468 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/A in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,468 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/01c3b9b3f30e4fb3bc409675e26f0353, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/a2cdf3a4de0245398c9a179fd86c0c63, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/c092ec43c55f4a8d98eb0fa39d0970c2] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=35.5 K 2024-12-05T12:15:36,468 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/c2ba0d4fef7b4c869e25f47d3dd133c2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/163705d33a8941fc906730d55006ba6d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/071fd700dec8474d93f1bda26d0ab6f0] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=37.8 K 2024-12-05T12:15:36,469 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 01c3b9b3f30e4fb3bc409675e26f0353, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733400930608 2024-12-05T12:15:36,469 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ba0d4fef7b4c869e25f47d3dd133c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733400930608 2024-12-05T12:15:36,469 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting a2cdf3a4de0245398c9a179fd86c0c63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733400932872 2024-12-05T12:15:36,469 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163705d33a8941fc906730d55006ba6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733400932872 2024-12-05T12:15:36,469 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 071fd700dec8474d93f1bda26d0ab6f0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733400934056 2024-12-05T12:15:36,469 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting c092ec43c55f4a8d98eb0fa39d0970c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733400934056 2024-12-05T12:15:36,475 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#B#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:36,475 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/9bd7ac543bcb444792be4f1c22e33695 is 50, key is test_row_0/B:col10/1733400934056/Put/seqid=0 2024-12-05T12:15:36,480 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#A#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:36,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742279_1455 (size=12459) 2024-12-05T12:15:36,481 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/7a31452edf92406e8edf8ab07f7b3bd8 is 50, key is test_row_0/A:col10/1733400934056/Put/seqid=0 2024-12-05T12:15:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:36,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742280_1456 (size=12459) 2024-12-05T12:15:36,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-05T12:15:36,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:36,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:36,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:36,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:36,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:36,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:36,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/e90f8ba5d2844d51952fdc9c1b905f8f is 50, key is test_row_0/A:col10/1733400935336/Put/seqid=0 2024-12-05T12:15:36,493 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/9bd7ac543bcb444792be4f1c22e33695 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9bd7ac543bcb444792be4f1c22e33695 2024-12-05T12:15:36,498 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/7a31452edf92406e8edf8ab07f7b3bd8 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/7a31452edf92406e8edf8ab07f7b3bd8 2024-12-05T12:15:36,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742281_1457 (size=14541) 2024-12-05T12:15:36,504 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/B of 7a7e5a7bd5645c444a2a620ce27fbe6a into 9bd7ac543bcb444792be4f1c22e33695(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:36,504 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:36,504 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/B, priority=13, startTime=1733400936467; duration=0sec 2024-12-05T12:15:36,504 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:36,504 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:B 2024-12-05T12:15:36,504 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:36,505 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:36,505 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/C is initiating minor compaction (all files) 2024-12-05T12:15:36,506 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/C in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,506 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7123cde97a204829a1906777462cd10a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5a13006acd3f4931a5cc33b56be5c5b1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/c612f36abfbb417c911672c573409eca] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=35.5 K 2024-12-05T12:15:36,507 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 7123cde97a204829a1906777462cd10a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733400930608 2024-12-05T12:15:36,507 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a13006acd3f4931a5cc33b56be5c5b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733400932872 2024-12-05T12:15:36,507 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting c612f36abfbb417c911672c573409eca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733400934056 2024-12-05T12:15:36,508 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/A of 7a7e5a7bd5645c444a2a620ce27fbe6a into 7a31452edf92406e8edf8ab07f7b3bd8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:36,508 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:36,508 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/A, priority=13, startTime=1733400936467; duration=0sec 2024-12-05T12:15:36,508 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:36,508 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:A 2024-12-05T12:15:36,512 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#C#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:36,512 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/5e2b73c62e904dc483c83b3376f7da61 is 50, key is test_row_0/C:col10/1733400934056/Put/seqid=0 2024-12-05T12:15:36,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742282_1458 (size=12459) 2024-12-05T12:15:36,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400996521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:36,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400996532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:36,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:36,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:36,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400996642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:36,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400996653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:36,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:36,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:36,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:36,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400996852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:36,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400996862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:36,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/e90f8ba5d2844d51952fdc9c1b905f8f 2024-12-05T12:15:36,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/17144d98860c4800841a551840dc8647 is 50, key is test_row_0/B:col10/1733400935336/Put/seqid=0 2024-12-05T12:15:36,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742283_1459 (size=12151) 2024-12-05T12:15:36,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/17144d98860c4800841a551840dc8647 2024-12-05T12:15:36,931 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/5e2b73c62e904dc483c83b3376f7da61 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5e2b73c62e904dc483c83b3376f7da61 2024-12-05T12:15:36,935 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/C of 7a7e5a7bd5645c444a2a620ce27fbe6a into 5e2b73c62e904dc483c83b3376f7da61(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:36,935 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:36,935 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/C, priority=13, startTime=1733400936467; duration=0sec 2024-12-05T12:15:36,935 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:36,936 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:C 2024-12-05T12:15:36,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/e1dc3e955c1a45ba8b1883c160bf718c is 50, key is test_row_0/C:col10/1733400935336/Put/seqid=0 2024-12-05T12:15:36,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742284_1460 (size=12151) 2024-12-05T12:15:36,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:36,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733400996982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:36,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733400996983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:36,986 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:36,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:36,986 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:36,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733400996993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:36,998 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:37,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:37,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:37,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:37,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:37,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:37,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:37,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:37,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:37,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:37,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:37,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:37,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400997162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:37,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400997173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:37,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:37,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:37,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T12:15:37,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/e1dc3e955c1a45ba8b1883c160bf718c 2024-12-05T12:15:37,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/e90f8ba5d2844d51952fdc9c1b905f8f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e90f8ba5d2844d51952fdc9c1b905f8f 2024-12-05T12:15:37,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e90f8ba5d2844d51952fdc9c1b905f8f, entries=200, sequenceid=178, filesize=14.2 K 2024-12-05T12:15:37,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/17144d98860c4800841a551840dc8647 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/17144d98860c4800841a551840dc8647 2024-12-05T12:15:37,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/17144d98860c4800841a551840dc8647, entries=150, sequenceid=178, filesize=11.9 K 2024-12-05T12:15:37,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/e1dc3e955c1a45ba8b1883c160bf718c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/e1dc3e955c1a45ba8b1883c160bf718c 2024-12-05T12:15:37,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/e1dc3e955c1a45ba8b1883c160bf718c, entries=150, sequenceid=178, filesize=11.9 K 2024-12-05T12:15:37,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 871ms, sequenceid=178, compaction requested=false 2024-12-05T12:15:37,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:37,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T12:15:37,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:37,460 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:15:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:37,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/6a6edcfb71f5418d8c29929f6e2c5791 is 50, key is test_row_0/A:col10/1733400936520/Put/seqid=0 2024-12-05T12:15:37,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742285_1461 (size=12151) 2024-12-05T12:15:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:37,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:37,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400937725, max timestamp=1733400937741Num uniques 859; 2024-12-05T12:15:37,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400937725, max timestamp=1733400937741Num uniques 859; 2024-12-05T12:15:37,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:37,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:37,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400937725, max timestamp=1733400937741Num uniques 859; 2024-12-05T12:15:37,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:37,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400997793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:37,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400997796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:37,870 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/6a6edcfb71f5418d8c29929f6e2c5791 2024-12-05T12:15:37,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/824423ccc1654c1187ba38e1e9649bc0 is 50, key is test_row_0/B:col10/1733400936520/Put/seqid=0 2024-12-05T12:15:37,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742286_1462 (size=12151) 2024-12-05T12:15:37,879 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/824423ccc1654c1187ba38e1e9649bc0 2024-12-05T12:15:37,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/0a7336acbc644bf48582bab1c2bf3379 is 50, key is test_row_0/C:col10/1733400936520/Put/seqid=0 2024-12-05T12:15:37,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742287_1463 (size=12151) 2024-12-05T12:15:37,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400997902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:37,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400997912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:38,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400998123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:38,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400998123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:38,287 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/0a7336acbc644bf48582bab1c2bf3379 2024-12-05T12:15:38,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/6a6edcfb71f5418d8c29929f6e2c5791 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6a6edcfb71f5418d8c29929f6e2c5791 2024-12-05T12:15:38,293 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6a6edcfb71f5418d8c29929f6e2c5791, entries=150, sequenceid=191, filesize=11.9 K 2024-12-05T12:15:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/824423ccc1654c1187ba38e1e9649bc0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/824423ccc1654c1187ba38e1e9649bc0 2024-12-05T12:15:38,296 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/824423ccc1654c1187ba38e1e9649bc0, entries=150, sequenceid=191, filesize=11.9 K 2024-12-05T12:15:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/0a7336acbc644bf48582bab1c2bf3379 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/0a7336acbc644bf48582bab1c2bf3379 2024-12-05T12:15:38,299 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/0a7336acbc644bf48582bab1c2bf3379, entries=150, sequenceid=191, filesize=11.9 K 2024-12-05T12:15:38,300 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 841ms, sequenceid=191, compaction requested=true 2024-12-05T12:15:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-05T12:15:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-05T12:15:38,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-05T12:15:38,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1360 sec 2024-12-05T12:15:38,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 3.1400 sec 2024-12-05T12:15:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:38,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-12-05T12:15:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:38,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/4646ac193fb74924a30736b4275ec39c is 50, key is test_row_0/A:col10/1733400937779/Put/seqid=0 2024-12-05T12:15:38,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742288_1464 (size=12151) 2024-12-05T12:15:38,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400998478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:38,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400998488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:38,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400998593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:38,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400998602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:38,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400998803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:38,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:38,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400998813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:38,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/4646ac193fb74924a30736b4275ec39c 2024-12-05T12:15:38,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/b9ea047fef114951bbc7cf653a1bd387 is 50, key is test_row_0/B:col10/1733400937779/Put/seqid=0 2024-12-05T12:15:38,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742289_1465 (size=12151) 2024-12-05T12:15:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400999112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:39,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733400999132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:39,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/b9ea047fef114951bbc7cf653a1bd387 2024-12-05T12:15:39,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/106dc2fadb9c42ccbd501bdeb69ebc75 is 50, key is test_row_0/C:col10/1733400937779/Put/seqid=0 2024-12-05T12:15:39,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742290_1466 (size=12151) 2024-12-05T12:15:39,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/106dc2fadb9c42ccbd501bdeb69ebc75 2024-12-05T12:15:39,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/4646ac193fb74924a30736b4275ec39c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/4646ac193fb74924a30736b4275ec39c 2024-12-05T12:15:39,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/4646ac193fb74924a30736b4275ec39c, entries=150, sequenceid=230, filesize=11.9 K 2024-12-05T12:15:39,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/b9ea047fef114951bbc7cf653a1bd387 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/b9ea047fef114951bbc7cf653a1bd387 2024-12-05T12:15:39,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/b9ea047fef114951bbc7cf653a1bd387, entries=150, sequenceid=230, filesize=11.9 K 2024-12-05T12:15:39,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/106dc2fadb9c42ccbd501bdeb69ebc75 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/106dc2fadb9c42ccbd501bdeb69ebc75 2024-12-05T12:15:39,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/106dc2fadb9c42ccbd501bdeb69ebc75, entries=150, sequenceid=230, filesize=11.9 K 2024-12-05T12:15:39,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 843ms, sequenceid=230, compaction requested=true 2024-12-05T12:15:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:39,294 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:15:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:39,294 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:15:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:39,297 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:15:39,297 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/B is initiating minor compaction (all files) 2024-12-05T12:15:39,297 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/B in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:39,297 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9bd7ac543bcb444792be4f1c22e33695, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/17144d98860c4800841a551840dc8647, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/824423ccc1654c1187ba38e1e9649bc0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/b9ea047fef114951bbc7cf653a1bd387] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=47.8 K 2024-12-05T12:15:39,297 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51302 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:15:39,298 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/A is initiating minor compaction (all files) 2024-12-05T12:15:39,298 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/A in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:39,298 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/7a31452edf92406e8edf8ab07f7b3bd8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e90f8ba5d2844d51952fdc9c1b905f8f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6a6edcfb71f5418d8c29929f6e2c5791, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/4646ac193fb74924a30736b4275ec39c] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=50.1 K 2024-12-05T12:15:39,298 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bd7ac543bcb444792be4f1c22e33695, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733400934056 2024-12-05T12:15:39,298 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a31452edf92406e8edf8ab07f7b3bd8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733400934056 2024-12-05T12:15:39,298 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 17144d98860c4800841a551840dc8647, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733400935327 2024-12-05T12:15:39,298 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting e90f8ba5d2844d51952fdc9c1b905f8f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733400935320 2024-12-05T12:15:39,299 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 824423ccc1654c1187ba38e1e9649bc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733400936518 2024-12-05T12:15:39,300 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a6edcfb71f5418d8c29929f6e2c5791, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733400936518 2024-12-05T12:15:39,300 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ea047fef114951bbc7cf653a1bd387, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733400937779 2024-12-05T12:15:39,300 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4646ac193fb74924a30736b4275ec39c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733400937779 2024-12-05T12:15:39,308 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:39,309 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/2a2023e54daf44bbbdde29e315e6375b is 50, key is test_row_0/A:col10/1733400937779/Put/seqid=0 2024-12-05T12:15:39,310 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:39,310 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/648e5aa01f46446c8b6cc4acbf42b82b is 50, key is test_row_0/B:col10/1733400937779/Put/seqid=0 2024-12-05T12:15:39,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742291_1467 (size=12595) 2024-12-05T12:15:39,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742292_1468 (size=12595) 2024-12-05T12:15:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T12:15:39,333 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:39,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-05T12:15:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T12:15:39,335 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:39,336 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:39,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T12:15:39,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T12:15:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:39,488 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T12:15:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:39,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/2adee0cfbde345e6beec32b344f609a2 is 50, key is test_row_1/A:col10/1733400938476/Put/seqid=0 2024-12-05T12:15:39,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742293_1469 (size=9757) 2024-12-05T12:15:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:39,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T12:15:39,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400939667, max timestamp=1733400939696Num uniques 859; 2024-12-05T12:15:39,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:15:39,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400939667, max timestamp=1733400939696Num uniques 859; 2024-12-05T12:15:39,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:15:39,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733400939667, max timestamp=1733400939696Num uniques 859; 2024-12-05T12:15:39,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T12:15:39,717 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/2a2023e54daf44bbbdde29e315e6375b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2a2023e54daf44bbbdde29e315e6375b 2024-12-05T12:15:39,719 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/648e5aa01f46446c8b6cc4acbf42b82b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/648e5aa01f46446c8b6cc4acbf42b82b 2024-12-05T12:15:39,724 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/A of 7a7e5a7bd5645c444a2a620ce27fbe6a into 2a2023e54daf44bbbdde29e315e6375b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:39,724 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:39,724 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/A, priority=12, startTime=1733400939294; duration=0sec 2024-12-05T12:15:39,724 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:39,724 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:A 2024-12-05T12:15:39,724 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:15:39,724 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/B of 7a7e5a7bd5645c444a2a620ce27fbe6a into 648e5aa01f46446c8b6cc4acbf42b82b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:39,724 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:39,724 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/B, priority=12, startTime=1733400939294; duration=0sec 2024-12-05T12:15:39,724 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:39,724 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:B 2024-12-05T12:15:39,725 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:15:39,725 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/C is initiating minor compaction (all files) 2024-12-05T12:15:39,725 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/C in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:39,725 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5e2b73c62e904dc483c83b3376f7da61, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/e1dc3e955c1a45ba8b1883c160bf718c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/0a7336acbc644bf48582bab1c2bf3379, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/106dc2fadb9c42ccbd501bdeb69ebc75] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=47.8 K 2024-12-05T12:15:39,725 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e2b73c62e904dc483c83b3376f7da61, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733400934056 2024-12-05T12:15:39,725 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1dc3e955c1a45ba8b1883c160bf718c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733400935327 2024-12-05T12:15:39,725 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a7336acbc644bf48582bab1c2bf3379, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733400936518 2024-12-05T12:15:39,726 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 106dc2fadb9c42ccbd501bdeb69ebc75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733400937779 2024-12-05T12:15:39,731 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:39,731 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/7c8d8a9a3f77455e88e517baeab9ab36 is 50, key is test_row_0/C:col10/1733400937779/Put/seqid=0 2024-12-05T12:15:39,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742294_1470 (size=12595) 2024-12-05T12:15:39,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733400939743, max timestamp=1733400939768Num uniques 1068; 2024-12-05T12:15:39,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-05T12:15:39,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733400939743, max timestamp=1733400939768Num uniques 1068; 2024-12-05T12:15:39,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733400939743, max timestamp=1733400939768Num uniques 1068; 2024-12-05T12:15:39,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-05T12:15:39,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-05T12:15:39,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733400939814, max timestamp=1733400939842Num uniques 1077; 2024-12-05T12:15:39,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-05T12:15:39,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733400939814, max timestamp=1733400939842Num uniques 1077; 2024-12-05T12:15:39,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-05T12:15:39,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733400939814, max timestamp=1733400939842Num uniques 1077; 2024-12-05T12:15:39,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-05T12:15:39,896 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/2adee0cfbde345e6beec32b344f609a2 2024-12-05T12:15:39,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/546105c4c4ff44d188abe6ad82d0a63e is 50, key is test_row_1/B:col10/1733400938476/Put/seqid=0 2024-12-05T12:15:39,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742295_1471 (size=9757) 2024-12-05T12:15:39,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733400939880, max timestamp=1733400939902Num uniques 1086; 2024-12-05T12:15:39,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-05T12:15:39,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733400939880, max timestamp=1733400939902Num uniques 1086; 2024-12-05T12:15:39,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-05T12:15:39,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733400939880, max timestamp=1733400939902Num uniques 1086; 2024-12-05T12:15:39,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-05T12:15:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T12:15:39,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733400999976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:39,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733400939963, max timestamp=1733400939984Num uniques 1095; 2024-12-05T12:15:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:39,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-05T12:15:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:39,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733400939963, max timestamp=1733400939984Num uniques 1095; 2024-12-05T12:15:39,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-05T12:15:39,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733400939963, max timestamp=1733400939984Num uniques 1095; 2024-12-05T12:15:39,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-05T12:15:40,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401000109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:40,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733400940096, max timestamp=1733400940117Num uniques 1054; 2024-12-05T12:15:40,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733400940096, max timestamp=1733400940117Num uniques 1104; 2024-12-05T12:15:40,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733400940096, max timestamp=1733400940117Num uniques 1104; 2024-12-05T12:15:40,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-05T12:15:40,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-05T12:15:40,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-05T12:15:40,138 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/7c8d8a9a3f77455e88e517baeab9ab36 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7c8d8a9a3f77455e88e517baeab9ab36 2024-12-05T12:15:40,142 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/C of 7a7e5a7bd5645c444a2a620ce27fbe6a into 7c8d8a9a3f77455e88e517baeab9ab36(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:40,142 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:40,142 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/C, priority=12, startTime=1733400939294; duration=0sec 2024-12-05T12:15:40,142 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:40,142 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:C 2024-12-05T12:15:40,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733400940184, max timestamp=1733400940198Num uniques 1013; 2024-12-05T12:15:40,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733400940184, max timestamp=1733400940198Num uniques 1013; 2024-12-05T12:15:40,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-05T12:15:40,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733400940184, max timestamp=1733400940198Num uniques 1013; 2024-12-05T12:15:40,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-05T12:15:40,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-05T12:15:40,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733400940245, max timestamp=1733400940261Num uniques 1022; 2024-12-05T12:15:40,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-05T12:15:40,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733400940245, max timestamp=1733400940261Num uniques 1022; 2024-12-05T12:15:40,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733400940245, max timestamp=1733400940261Num uniques 1022; 2024-12-05T12:15:40,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-05T12:15:40,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-05T12:15:40,304 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/546105c4c4ff44d188abe6ad82d0a63e 2024-12-05T12:15:40,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/b994213ea30b4055835133fd0d3158fc is 50, key is test_row_1/C:col10/1733400938476/Put/seqid=0 2024-12-05T12:15:40,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742296_1472 (size=9757) 2024-12-05T12:15:40,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733400940347, max timestamp=1733400940362Num uniques 1031; 2024-12-05T12:15:40,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733400940347, max timestamp=1733400940362Num uniques 1031; 2024-12-05T12:15:40,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-05T12:15:40,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-05T12:15:40,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733400940347, max timestamp=1733400940362Num uniques 1031; 2024-12-05T12:15:40,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-05T12:15:40,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733400940399, max timestamp=1733400940429Num uniques 1040; 2024-12-05T12:15:40,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733400940399, max timestamp=1733400940429Num uniques 1040; 2024-12-05T12:15:40,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733400940399, max timestamp=1733400940429Num uniques 1040; 2024-12-05T12:15:40,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-05T12:15:40,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-05T12:15:40,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-05T12:15:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T12:15:40,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401000487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:40,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733400940473, max timestamp=1733400940489Num uniques 1099; 2024-12-05T12:15:40,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-12-05T12:15:40,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733400940473, max timestamp=1733400940489Num uniques 1099; 2024-12-05T12:15:40,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-12-05T12:15:40,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733400940473, max timestamp=1733400940489Num uniques 1099; 2024-12-05T12:15:40,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-12-05T12:15:40,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401000578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401000603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:40,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401000692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:40,718 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/b994213ea30b4055835133fd0d3158fc 2024-12-05T12:15:40,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/2adee0cfbde345e6beec32b344f609a2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2adee0cfbde345e6beec32b344f609a2 2024-12-05T12:15:40,726 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2adee0cfbde345e6beec32b344f609a2, entries=100, sequenceid=238, filesize=9.5 K 2024-12-05T12:15:40,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/546105c4c4ff44d188abe6ad82d0a63e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/546105c4c4ff44d188abe6ad82d0a63e 2024-12-05T12:15:40,730 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/546105c4c4ff44d188abe6ad82d0a63e, entries=100, sequenceid=238, filesize=9.5 K 2024-12-05T12:15:40,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/b994213ea30b4055835133fd0d3158fc as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b994213ea30b4055835133fd0d3158fc 2024-12-05T12:15:40,734 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b994213ea30b4055835133fd0d3158fc, entries=100, sequenceid=238, filesize=9.5 K 2024-12-05T12:15:40,734 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 1246ms, sequenceid=238, compaction requested=false 2024-12-05T12:15:40,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:40,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:40,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-05T12:15:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-05T12:15:40,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-05T12:15:40,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-12-05T12:15:40,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4050 sec 2024-12-05T12:15:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:40,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=154.31 KB heapSize=437.02 KB 2024-12-05T12:15:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:40,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/32f038ffc026492a88b80f2d9b87e98e is 50, key is test_row_0/A:col10/1733400940570/Put/seqid=0 2024-12-05T12:15:40,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742297_1473 (size=14741) 2024-12-05T12:15:40,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401000836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:40,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401000903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:40,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401000942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:41,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401001152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:41,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401001213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:41,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/32f038ffc026492a88b80f2d9b87e98e 2024-12-05T12:15:41,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/bfcaffdcd0c746b29136bf320f2abda6 is 50, key is test_row_0/B:col10/1733400940570/Put/seqid=0 2024-12-05T12:15:41,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742298_1474 (size=12301) 2024-12-05T12:15:41,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/bfcaffdcd0c746b29136bf320f2abda6 2024-12-05T12:15:41,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/b3a9e4e7d25e41c5963802f1c5a3b5f2 is 50, key is test_row_0/C:col10/1733400940570/Put/seqid=0 2024-12-05T12:15:41,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742299_1475 (size=12301) 2024-12-05T12:15:41,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401001473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T12:15:41,483 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:41,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-05T12:15:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T12:15:41,485 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:41,485 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:41,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T12:15:41,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:41,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:41,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/b3a9e4e7d25e41c5963802f1c5a3b5f2 2024-12-05T12:15:41,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/32f038ffc026492a88b80f2d9b87e98e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/32f038ffc026492a88b80f2d9b87e98e 2024-12-05T12:15:41,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/32f038ffc026492a88b80f2d9b87e98e, entries=200, sequenceid=451, filesize=14.4 K 2024-12-05T12:15:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/bfcaffdcd0c746b29136bf320f2abda6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bfcaffdcd0c746b29136bf320f2abda6 2024-12-05T12:15:41,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bfcaffdcd0c746b29136bf320f2abda6, entries=150, sequenceid=451, filesize=12.0 K 2024-12-05T12:15:41,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/b3a9e4e7d25e41c5963802f1c5a3b5f2 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b3a9e4e7d25e41c5963802f1c5a3b5f2 2024-12-05T12:15:41,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b3a9e4e7d25e41c5963802f1c5a3b5f2, entries=150, sequenceid=451, filesize=12.0 K 2024-12-05T12:15:41,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.97 KB/447456, currentSize=26.84 KB/27480 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 842ms, sequenceid=451, compaction requested=true 2024-12-05T12:15:41,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:41,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:41,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:41,656 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:41,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:41,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:41,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:41,656 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:41,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:41,657 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:41,657 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/B is initiating minor compaction (all files) 2024-12-05T12:15:41,657 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/B in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:41,657 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/648e5aa01f46446c8b6cc4acbf42b82b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/546105c4c4ff44d188abe6ad82d0a63e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bfcaffdcd0c746b29136bf320f2abda6] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=33.8 K 2024-12-05T12:15:41,657 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 648e5aa01f46446c8b6cc4acbf42b82b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733400937779 2024-12-05T12:15:41,657 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:41,658 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/A is initiating minor compaction (all files) 2024-12-05T12:15:41,658 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 546105c4c4ff44d188abe6ad82d0a63e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733400938476 2024-12-05T12:15:41,658 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/A in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:41,658 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2a2023e54daf44bbbdde29e315e6375b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2adee0cfbde345e6beec32b344f609a2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/32f038ffc026492a88b80f2d9b87e98e] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=36.2 K 2024-12-05T12:15:41,658 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting bfcaffdcd0c746b29136bf320f2abda6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733400940544 2024-12-05T12:15:41,658 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a2023e54daf44bbbdde29e315e6375b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733400937779 2024-12-05T12:15:41,658 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2adee0cfbde345e6beec32b344f609a2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733400938476 2024-12-05T12:15:41,659 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32f038ffc026492a88b80f2d9b87e98e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733400940544 2024-12-05T12:15:41,665 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#B#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:41,665 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#A#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:41,666 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/bd5ff75cf8a14e7484f0b5c98f30d5f3 is 50, key is test_row_0/B:col10/1733400940570/Put/seqid=0 2024-12-05T12:15:41,666 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/ce24aa11a53c429e82ad18e99c9d2c09 is 50, key is test_row_0/A:col10/1733400940570/Put/seqid=0 2024-12-05T12:15:41,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742301_1477 (size=12847) 2024-12-05T12:15:41,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742300_1476 (size=12847) 2024-12-05T12:15:41,676 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/ce24aa11a53c429e82ad18e99c9d2c09 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ce24aa11a53c429e82ad18e99c9d2c09 2024-12-05T12:15:41,680 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/A of 7a7e5a7bd5645c444a2a620ce27fbe6a into ce24aa11a53c429e82ad18e99c9d2c09(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:41,680 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:41,680 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/A, priority=13, startTime=1733400941656; duration=0sec 2024-12-05T12:15:41,681 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:41,681 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:A 2024-12-05T12:15:41,681 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:41,682 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:41,682 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/C is initiating minor compaction (all files) 2024-12-05T12:15:41,682 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/C in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:41,682 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7c8d8a9a3f77455e88e517baeab9ab36, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b994213ea30b4055835133fd0d3158fc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b3a9e4e7d25e41c5963802f1c5a3b5f2] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=33.8 K 2024-12-05T12:15:41,682 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c8d8a9a3f77455e88e517baeab9ab36, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733400937779 2024-12-05T12:15:41,684 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b994213ea30b4055835133fd0d3158fc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733400938476 2024-12-05T12:15:41,684 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3a9e4e7d25e41c5963802f1c5a3b5f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733400940544 2024-12-05T12:15:41,691 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#C#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:41,691 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/aec55dade5c14bbd831f335c9083f77f is 50, key is test_row_0/C:col10/1733400940570/Put/seqid=0 2024-12-05T12:15:41,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742302_1478 (size=12847) 2024-12-05T12:15:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:41,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:15:41,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:41,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:41,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:41,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:41,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:41,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:41,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/3941dd157252437893e96b6b73ef1c49 is 50, key is test_row_0/A:col10/1733400941733/Put/seqid=0 2024-12-05T12:15:41,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742303_1479 (size=12297) 2024-12-05T12:15:41,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:41,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:41,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:41,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:41,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:41,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T12:15:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400941818, max timestamp=1733400941847Num uniques 859; 2024-12-05T12:15:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400941818, max timestamp=1733400941847Num uniques 859; 2024-12-05T12:15:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400941818, max timestamp=1733400941847Num uniques 859; 2024-12-05T12:15:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:41,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:41,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:41,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401001935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:41,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401001983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401002053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:42,073 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/bd5ff75cf8a14e7484f0b5c98f30d5f3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bd5ff75cf8a14e7484f0b5c98f30d5f3 2024-12-05T12:15:42,076 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/B of 7a7e5a7bd5645c444a2a620ce27fbe6a into bd5ff75cf8a14e7484f0b5c98f30d5f3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:42,077 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:42,077 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/B, priority=13, startTime=1733400941656; duration=0sec 2024-12-05T12:15:42,077 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:42,077 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:B 2024-12-05T12:15:42,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:42,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:42,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,100 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/aec55dade5c14bbd831f335c9083f77f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/aec55dade5c14bbd831f335c9083f77f 2024-12-05T12:15:42,103 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/C of 7a7e5a7bd5645c444a2a620ce27fbe6a into aec55dade5c14bbd831f335c9083f77f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:42,103 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:42,103 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/C, priority=13, startTime=1733400941656; duration=0sec 2024-12-05T12:15:42,103 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:42,103 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:C 2024-12-05T12:15:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T12:15:42,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/3941dd157252437893e96b6b73ef1c49 2024-12-05T12:15:42,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/0dd110ede7454ff884daeb19b2ee930a is 50, key is test_row_0/B:col10/1733400941733/Put/seqid=0 2024-12-05T12:15:42,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742304_1480 (size=9857) 2024-12-05T12:15:42,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:42,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:42,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401002262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:42,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:42,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:42,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/0dd110ede7454ff884daeb19b2ee930a 2024-12-05T12:15:42,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/66096e2c03bd4d89bce75d08047a6049 is 50, key is test_row_0/C:col10/1733400941733/Put/seqid=0 2024-12-05T12:15:42,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742305_1481 (size=9857) 2024-12-05T12:15:42,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401002573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T12:15:42,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:42,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:42,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:42,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:42,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:42,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:42,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/66096e2c03bd4d89bce75d08047a6049 2024-12-05T12:15:42,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/3941dd157252437893e96b6b73ef1c49 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3941dd157252437893e96b6b73ef1c49 2024-12-05T12:15:42,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3941dd157252437893e96b6b73ef1c49, entries=150, sequenceid=462, filesize=12.0 K 2024-12-05T12:15:42,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/0dd110ede7454ff884daeb19b2ee930a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/0dd110ede7454ff884daeb19b2ee930a 2024-12-05T12:15:42,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/0dd110ede7454ff884daeb19b2ee930a, entries=100, sequenceid=462, filesize=9.6 K 2024-12-05T12:15:42,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/66096e2c03bd4d89bce75d08047a6049 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/66096e2c03bd4d89bce75d08047a6049 2024-12-05T12:15:42,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/66096e2c03bd4d89bce75d08047a6049, entries=100, sequenceid=462, filesize=9.6 K 2024-12-05T12:15:42,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 1248ms, sequenceid=462, compaction requested=false 2024-12-05T12:15:42,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:42,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-05T12:15:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:42,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:42,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:42,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:43,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/6d062abb8a9a4dbf944495d168114ee1 is 50, key is test_row_0/A:col10/1733400942994/Put/seqid=0 2024-12-05T12:15:43,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742306_1482 (size=14741) 2024-12-05T12:15:43,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/6d062abb8a9a4dbf944495d168114ee1 2024-12-05T12:15:43,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/e568f55b4ea64792b73bbe4e99e34f60 is 50, key is test_row_0/B:col10/1733400942994/Put/seqid=0 2024-12-05T12:15:43,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:43,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:43,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742307_1483 (size=12301) 2024-12-05T12:15:43,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401003035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:43,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401003093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:43,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401003153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:43,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:43,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:43,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401003362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:43,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/e568f55b4ea64792b73bbe4e99e34f60 2024-12-05T12:15:43,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/08cf852574f04d32a650dd974bb9aad5 is 50, key is test_row_0/C:col10/1733400942994/Put/seqid=0 2024-12-05T12:15:43,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742308_1484 (size=12301) 2024-12-05T12:15:43,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:43,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:43,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:43,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:43,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T12:15:43,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401003672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:43,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:43,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:43,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:43,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/08cf852574f04d32a650dd974bb9aad5 2024-12-05T12:15:43,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/6d062abb8a9a4dbf944495d168114ee1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6d062abb8a9a4dbf944495d168114ee1 2024-12-05T12:15:43,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6d062abb8a9a4dbf944495d168114ee1, entries=200, sequenceid=502, filesize=14.4 K 2024-12-05T12:15:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/e568f55b4ea64792b73bbe4e99e34f60 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/e568f55b4ea64792b73bbe4e99e34f60 2024-12-05T12:15:43,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/e568f55b4ea64792b73bbe4e99e34f60, entries=150, sequenceid=502, filesize=12.0 K 2024-12-05T12:15:43,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/08cf852574f04d32a650dd974bb9aad5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/08cf852574f04d32a650dd974bb9aad5 2024-12-05T12:15:43,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/08cf852574f04d32a650dd974bb9aad5, entries=150, sequenceid=502, filesize=12.0 K 2024-12-05T12:15:43,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 844ms, sequenceid=502, compaction requested=true 2024-12-05T12:15:43,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:43,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:43,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:43,840 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:43,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:43,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:43,840 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:43,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:43,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:43,841 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39885 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:43,841 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:43,841 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/B is initiating minor compaction (all files) 2024-12-05T12:15:43,841 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/A is initiating minor compaction (all files) 2024-12-05T12:15:43,841 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/B in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,841 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/A in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,841 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ce24aa11a53c429e82ad18e99c9d2c09, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3941dd157252437893e96b6b73ef1c49, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6d062abb8a9a4dbf944495d168114ee1] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=39.0 K 2024-12-05T12:15:43,841 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bd5ff75cf8a14e7484f0b5c98f30d5f3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/0dd110ede7454ff884daeb19b2ee930a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/e568f55b4ea64792b73bbe4e99e34f60] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=34.2 K 2024-12-05T12:15:43,841 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting bd5ff75cf8a14e7484f0b5c98f30d5f3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733400940544 2024-12-05T12:15:43,841 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce24aa11a53c429e82ad18e99c9d2c09, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733400940544 2024-12-05T12:15:43,841 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd110ede7454ff884daeb19b2ee930a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733400941733 2024-12-05T12:15:43,841 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3941dd157252437893e96b6b73ef1c49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733400941733 2024-12-05T12:15:43,841 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting e568f55b4ea64792b73bbe4e99e34f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733400941917 2024-12-05T12:15:43,841 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d062abb8a9a4dbf944495d168114ee1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733400941917 2024-12-05T12:15:43,852 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#B#compaction#411 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:43,852 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/4bfbf2d0596e4821aec6b9e0d22a7d43 is 50, key is test_row_0/B:col10/1733400942994/Put/seqid=0 2024-12-05T12:15:43,853 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:43,853 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/e87fe22811a145c2ba75c77fa57f6a35 is 50, key is test_row_0/A:col10/1733400942994/Put/seqid=0 2024-12-05T12:15:43,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742310_1486 (size=12949) 2024-12-05T12:15:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742309_1485 (size=12949) 2024-12-05T12:15:43,865 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/e87fe22811a145c2ba75c77fa57f6a35 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e87fe22811a145c2ba75c77fa57f6a35 2024-12-05T12:15:43,869 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/A of 7a7e5a7bd5645c444a2a620ce27fbe6a into e87fe22811a145c2ba75c77fa57f6a35(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:43,869 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:43,869 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/A, priority=13, startTime=1733400943840; duration=0sec 2024-12-05T12:15:43,869 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:43,869 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:A 2024-12-05T12:15:43,869 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:43,869 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:43,870 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/C is initiating minor compaction (all files) 2024-12-05T12:15:43,870 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/C in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,870 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/aec55dade5c14bbd831f335c9083f77f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/66096e2c03bd4d89bce75d08047a6049, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/08cf852574f04d32a650dd974bb9aad5] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=34.2 K 2024-12-05T12:15:43,870 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting aec55dade5c14bbd831f335c9083f77f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733400940544 2024-12-05T12:15:43,870 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66096e2c03bd4d89bce75d08047a6049, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733400941733 2024-12-05T12:15:43,870 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08cf852574f04d32a650dd974bb9aad5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733400941917 2024-12-05T12:15:43,876 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:43,876 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/77968d7bbd9f4a44ab8d8829c58a82be is 50, key is test_row_0/C:col10/1733400942994/Put/seqid=0 2024-12-05T12:15:43,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742311_1487 (size=12949) 2024-12-05T12:15:43,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T12:15:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:43,928 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-05T12:15:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:43,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/032434045de9475eb8dac486319e5ae5 is 50, key is test_row_0/A:col10/1733400943027/Put/seqid=0 2024-12-05T12:15:43,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742312_1488 (size=12301) 2024-12-05T12:15:43,935 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/032434045de9475eb8dac486319e5ae5 2024-12-05T12:15:43,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/ab5fcec9a57045b38c3f33eea317f6af is 50, key is test_row_0/B:col10/1733400943027/Put/seqid=0 2024-12-05T12:15:43,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742313_1489 (size=12301) 2024-12-05T12:15:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:44,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:44,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400944175, max timestamp=1733400944204Num uniques 859; 2024-12-05T12:15:44,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400944175, max timestamp=1733400944204Num uniques 859; 2024-12-05T12:15:44,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400944175, max timestamp=1733400944204Num uniques 859; 2024-12-05T12:15:44,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:44,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:44,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:44,262 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/4bfbf2d0596e4821aec6b9e0d22a7d43 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4bfbf2d0596e4821aec6b9e0d22a7d43 2024-12-05T12:15:44,265 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/B of 7a7e5a7bd5645c444a2a620ce27fbe6a into 4bfbf2d0596e4821aec6b9e0d22a7d43(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:44,265 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:44,265 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/B, priority=13, startTime=1733400943840; duration=0sec 2024-12-05T12:15:44,265 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:44,265 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:B 2024-12-05T12:15:44,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733400944258, max timestamp=1733400944286Num uniques 1018; 2024-12-05T12:15:44,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-05T12:15:44,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733400944258, max timestamp=1733400944286Num uniques 1018; 2024-12-05T12:15:44,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733400944258, max timestamp=1733400944286Num uniques 1018; 2024-12-05T12:15:44,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-05T12:15:44,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-05T12:15:44,293 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/77968d7bbd9f4a44ab8d8829c58a82be as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/77968d7bbd9f4a44ab8d8829c58a82be 2024-12-05T12:15:44,296 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/C of 7a7e5a7bd5645c444a2a620ce27fbe6a into 77968d7bbd9f4a44ab8d8829c58a82be(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:44,296 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:44,296 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/C, priority=13, startTime=1733400943840; duration=0sec 2024-12-05T12:15:44,296 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:44,296 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:C 2024-12-05T12:15:44,345 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/ab5fcec9a57045b38c3f33eea317f6af 2024-12-05T12:15:44,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/acdf96d058e14845b099aa2131aa05a1 is 50, key is test_row_0/C:col10/1733400943027/Put/seqid=0 2024-12-05T12:15:44,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742314_1490 (size=12301) 2024-12-05T12:15:44,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401004345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:44,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401004354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:44,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401004462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:44,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401004472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:44,516 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T12:15:44,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401004673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401004683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:44,752 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/acdf96d058e14845b099aa2131aa05a1 2024-12-05T12:15:44,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/032434045de9475eb8dac486319e5ae5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/032434045de9475eb8dac486319e5ae5 2024-12-05T12:15:44,763 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/032434045de9475eb8dac486319e5ae5, entries=150, sequenceid=512, filesize=12.0 K 2024-12-05T12:15:44,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/ab5fcec9a57045b38c3f33eea317f6af as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab5fcec9a57045b38c3f33eea317f6af 2024-12-05T12:15:44,767 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab5fcec9a57045b38c3f33eea317f6af, entries=150, sequenceid=512, filesize=12.0 K 2024-12-05T12:15:44,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/acdf96d058e14845b099aa2131aa05a1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/acdf96d058e14845b099aa2131aa05a1 2024-12-05T12:15:44,770 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/acdf96d058e14845b099aa2131aa05a1, entries=150, sequenceid=512, filesize=12.0 K 2024-12-05T12:15:44,771 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 843ms, sequenceid=512, compaction requested=false 2024-12-05T12:15:44,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:44,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:44,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-05T12:15:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-05T12:15:44,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-05T12:15:44,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2870 sec 2024-12-05T12:15:44,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 3.2890 sec 2024-12-05T12:15:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:44,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-12-05T12:15:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T12:15:44,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/ccece7d347e84ce9ba1aa9f25a618630 is 50, key is test_row_0/A:col10/1733400944352/Put/seqid=0 2024-12-05T12:15:44,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742315_1491 (size=14741) 2024-12-05T12:15:45,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401005012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:45,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401005013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:45,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401005122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:45,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:45,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401005133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:45,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:45,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401005333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:45,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401005342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:45,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/ccece7d347e84ce9ba1aa9f25a618630 2024-12-05T12:15:45,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/d53909de1b07429986cb274a8a412197 is 50, key is test_row_0/B:col10/1733400944352/Put/seqid=0 2024-12-05T12:15:45,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742316_1492 (size=12301) 2024-12-05T12:15:45,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401005643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T12:15:45,653 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:45,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-05T12:15:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T12:15:45,655 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:45,656 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:45,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:45,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401005653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T12:15:45,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T12:15:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:45,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:45,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:45,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/d53909de1b07429986cb274a8a412197 2024-12-05T12:15:45,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/05ace6d36029424bbdfe50f493a3bde6 is 50, key is test_row_0/C:col10/1733400944352/Put/seqid=0 2024-12-05T12:15:45,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742317_1493 (size=12301) 2024-12-05T12:15:45,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T12:15:45,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:45,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:45,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:45,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:45,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T12:15:46,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T12:15:46,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:46,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:46,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:46,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:46,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:46,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401006153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:46,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401006172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:46,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:46,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:46,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:46,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/05ace6d36029424bbdfe50f493a3bde6 2024-12-05T12:15:46,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/ccece7d347e84ce9ba1aa9f25a618630 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ccece7d347e84ce9ba1aa9f25a618630 2024-12-05T12:15:46,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ccece7d347e84ce9ba1aa9f25a618630, entries=200, sequenceid=569, filesize=14.4 K 2024-12-05T12:15:46,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/d53909de1b07429986cb274a8a412197 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/d53909de1b07429986cb274a8a412197 2024-12-05T12:15:46,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/d53909de1b07429986cb274a8a412197, entries=150, sequenceid=569, filesize=12.0 K 2024-12-05T12:15:46,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/05ace6d36029424bbdfe50f493a3bde6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/05ace6d36029424bbdfe50f493a3bde6 2024-12-05T12:15:46,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/05ace6d36029424bbdfe50f493a3bde6, entries=150, sequenceid=569, filesize=12.0 K 2024-12-05T12:15:46,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 1247ms, sequenceid=569, compaction requested=true 2024-12-05T12:15:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:46,236 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:46,236 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:46,237 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:46,237 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:46,237 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/A is initiating minor compaction (all files) 2024-12-05T12:15:46,237 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/B is initiating minor compaction (all files) 2024-12-05T12:15:46,237 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/A in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:46,237 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/B in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:46,237 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e87fe22811a145c2ba75c77fa57f6a35, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/032434045de9475eb8dac486319e5ae5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ccece7d347e84ce9ba1aa9f25a618630] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=39.1 K 2024-12-05T12:15:46,237 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4bfbf2d0596e4821aec6b9e0d22a7d43, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab5fcec9a57045b38c3f33eea317f6af, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/d53909de1b07429986cb274a8a412197] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=36.7 K 2024-12-05T12:15:46,237 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting e87fe22811a145c2ba75c77fa57f6a35, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733400941917 2024-12-05T12:15:46,237 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bfbf2d0596e4821aec6b9e0d22a7d43, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733400941917 2024-12-05T12:15:46,237 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 032434045de9475eb8dac486319e5ae5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733400943006 2024-12-05T12:15:46,237 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5fcec9a57045b38c3f33eea317f6af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733400943006 2024-12-05T12:15:46,237 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccece7d347e84ce9ba1aa9f25a618630, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1733400944331 2024-12-05T12:15:46,238 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d53909de1b07429986cb274a8a412197, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1733400944331 2024-12-05T12:15:46,245 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:46,245 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/ab6eb7979c12409e8f40d5ef8cb8dd20 is 50, key is test_row_0/B:col10/1733400944352/Put/seqid=0 2024-12-05T12:15:46,248 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#A#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:46,249 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/42b3cb14155d4fc39b36360d3119852a is 50, key is test_row_0/A:col10/1733400944352/Put/seqid=0 2024-12-05T12:15:46,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742318_1494 (size=13051) 2024-12-05T12:15:46,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742319_1495 (size=13051) 2024-12-05T12:15:46,262 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/ab6eb7979c12409e8f40d5ef8cb8dd20 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab6eb7979c12409e8f40d5ef8cb8dd20 2024-12-05T12:15:46,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T12:15:46,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:46,267 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T12:15:46,269 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/B of 7a7e5a7bd5645c444a2a620ce27fbe6a into ab6eb7979c12409e8f40d5ef8cb8dd20(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:46,269 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:46,269 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/B, priority=13, startTime=1733400946236; duration=0sec 2024-12-05T12:15:46,269 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:46,269 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:B 2024-12-05T12:15:46,269 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:46,271 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:46,271 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/C is initiating minor compaction (all files) 2024-12-05T12:15:46,271 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/C in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:46,271 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/77968d7bbd9f4a44ab8d8829c58a82be, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/acdf96d058e14845b099aa2131aa05a1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/05ace6d36029424bbdfe50f493a3bde6] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=36.7 K 2024-12-05T12:15:46,273 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 77968d7bbd9f4a44ab8d8829c58a82be, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733400941917 2024-12-05T12:15:46,274 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting acdf96d058e14845b099aa2131aa05a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733400943006 2024-12-05T12:15:46,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/3748fa113f244044ae3a0214ef3894bf is 50, key is test_row_0/A:col10/1733400945011/Put/seqid=0 2024-12-05T12:15:46,275 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ace6d36029424bbdfe50f493a3bde6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1733400944331 2024-12-05T12:15:46,281 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#C#compaction#423 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:46,282 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/6f602aa9c12c4cc5b7d161da5758ac88 is 50, key is test_row_0/C:col10/1733400944352/Put/seqid=0 2024-12-05T12:15:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T12:15:46,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742320_1496 (size=13051) 2024-12-05T12:15:46,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742321_1497 (size=12301) 2024-12-05T12:15:46,662 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/42b3cb14155d4fc39b36360d3119852a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/42b3cb14155d4fc39b36360d3119852a 2024-12-05T12:15:46,665 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/A of 7a7e5a7bd5645c444a2a620ce27fbe6a into 42b3cb14155d4fc39b36360d3119852a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:46,665 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:46,665 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/A, priority=13, startTime=1733400946236; duration=0sec 2024-12-05T12:15:46,665 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:46,665 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:A 2024-12-05T12:15:46,691 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/6f602aa9c12c4cc5b7d161da5758ac88 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/6f602aa9c12c4cc5b7d161da5758ac88 2024-12-05T12:15:46,693 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/3748fa113f244044ae3a0214ef3894bf 2024-12-05T12:15:46,695 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/C of 7a7e5a7bd5645c444a2a620ce27fbe6a into 6f602aa9c12c4cc5b7d161da5758ac88(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:46,695 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:46,695 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/C, priority=13, startTime=1733400946236; duration=0sec 2024-12-05T12:15:46,695 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:46,695 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:C 2024-12-05T12:15:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/4557d3b6726a47ff8cf645e24776312c is 50, key is test_row_0/B:col10/1733400945011/Put/seqid=0 2024-12-05T12:15:46,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742322_1498 (size=12301) 2024-12-05T12:15:46,704 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/4557d3b6726a47ff8cf645e24776312c 2024-12-05T12:15:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/3416558eed834e7bae20f8b28a5d886f is 50, key is test_row_0/C:col10/1733400945011/Put/seqid=0 2024-12-05T12:15:46,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742323_1499 (size=12301) 2024-12-05T12:15:46,718 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/3416558eed834e7bae20f8b28a5d886f 2024-12-05T12:15:46,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/3748fa113f244044ae3a0214ef3894bf as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3748fa113f244044ae3a0214ef3894bf 2024-12-05T12:15:46,725 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3748fa113f244044ae3a0214ef3894bf, entries=150, sequenceid=578, filesize=12.0 K 2024-12-05T12:15:46,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/4557d3b6726a47ff8cf645e24776312c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4557d3b6726a47ff8cf645e24776312c 2024-12-05T12:15:46,728 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4557d3b6726a47ff8cf645e24776312c, entries=150, sequenceid=578, filesize=12.0 K 2024-12-05T12:15:46,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/3416558eed834e7bae20f8b28a5d886f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/3416558eed834e7bae20f8b28a5d886f 2024-12-05T12:15:46,732 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/3416558eed834e7bae20f8b28a5d886f, entries=150, sequenceid=578, filesize=12.0 K 2024-12-05T12:15:46,732 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 465ms, sequenceid=578, compaction requested=false 2024-12-05T12:15:46,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:46,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:46,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-05T12:15:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-05T12:15:46,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-05T12:15:46,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0800 sec 2024-12-05T12:15:46,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0840 sec 2024-12-05T12:15:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T12:15:46,793 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:46,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-05T12:15:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T12:15:46,796 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:46,797 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:46,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T12:15:46,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-05T12:15:46,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:46,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:46,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:46,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-05T12:15:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-05T12:15:46,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-05T12:15:46,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 153 msec 2024-12-05T12:15:46,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 161 msec 2024-12-05T12:15:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:47,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:15:47,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:47,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:47,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:47,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:47,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:47,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:47,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/bea4c30c2327436da10cfb74b48b8d98 is 50, key is test_row_0/A:col10/1733400947084/Put/seqid=0 2024-12-05T12:15:47,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742324_1500 (size=14741) 2024-12-05T12:15:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T12:15:47,113 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:47,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-05T12:15:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T12:15:47,115 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:47,115 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:47,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:47,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400947094, max timestamp=1733400947123Num uniques 859; 2024-12-05T12:15:47,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400947094, max timestamp=1733400947123Num uniques 859; 2024-12-05T12:15:47,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:15:47,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400947094, max timestamp=1733400947123Num uniques 859; 2024-12-05T12:15:47,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:15:47,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:15:47,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733401007152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733401007158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733401007160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49386 deadline: 1733401007173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,179 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49404 deadline: 1733401007183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,188 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 7a7e5a7bd5645c444a2a620ce27fbe6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T12:15:47,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-05T12:15:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733401007263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733401007273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733401007273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-05T12:15:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T12:15:47,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733401007473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733401007483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733401007483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/bea4c30c2327436da10cfb74b48b8d98 2024-12-05T12:15:47,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/32409f74d5ca40ea95389b1d0a5ee3b3 is 50, key is test_row_0/B:col10/1733400947084/Put/seqid=0 2024-12-05T12:15:47,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742325_1501 (size=12301) 2024-12-05T12:15:47,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-05T12:15:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-05T12:15:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T12:15:47,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733401007792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733401007792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733401007792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:47,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-05T12:15:47,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:47,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:47,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/32409f74d5ca40ea95389b1d0a5ee3b3 2024-12-05T12:15:47,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/30744fd21dff49e9821a185781835590 is 50, key is test_row_0/C:col10/1733400947084/Put/seqid=0 2024-12-05T12:15:47,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742326_1502 (size=12301) 2024-12-05T12:15:47,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/30744fd21dff49e9821a185781835590 2024-12-05T12:15:47,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/bea4c30c2327436da10cfb74b48b8d98 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/bea4c30c2327436da10cfb74b48b8d98 2024-12-05T12:15:47,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/bea4c30c2327436da10cfb74b48b8d98, entries=200, sequenceid=591, filesize=14.4 K 2024-12-05T12:15:47,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/32409f74d5ca40ea95389b1d0a5ee3b3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/32409f74d5ca40ea95389b1d0a5ee3b3 2024-12-05T12:15:47,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/32409f74d5ca40ea95389b1d0a5ee3b3, entries=150, sequenceid=591, filesize=12.0 K 2024-12-05T12:15:47,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/30744fd21dff49e9821a185781835590 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/30744fd21dff49e9821a185781835590 2024-12-05T12:15:47,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/30744fd21dff49e9821a185781835590, entries=150, sequenceid=591, filesize=12.0 K 2024-12-05T12:15:47,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 853ms, sequenceid=591, compaction requested=true 2024-12-05T12:15:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:47,938 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:47,938 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:15:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:47,939 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:47,939 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:47,939 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/A is initiating minor compaction (all files) 2024-12-05T12:15:47,939 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/B is initiating minor compaction (all files) 2024-12-05T12:15:47,939 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/B in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,939 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/A in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:47,939 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab6eb7979c12409e8f40d5ef8cb8dd20, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4557d3b6726a47ff8cf645e24776312c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/32409f74d5ca40ea95389b1d0a5ee3b3] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=36.8 K 2024-12-05T12:15:47,939 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/42b3cb14155d4fc39b36360d3119852a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3748fa113f244044ae3a0214ef3894bf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/bea4c30c2327436da10cfb74b48b8d98] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=39.2 K 2024-12-05T12:15:47,939 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6eb7979c12409e8f40d5ef8cb8dd20, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1733400944331 2024-12-05T12:15:47,939 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b3cb14155d4fc39b36360d3119852a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1733400944331 2024-12-05T12:15:47,939 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 4557d3b6726a47ff8cf645e24776312c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=578, earliestPutTs=1733400945000 2024-12-05T12:15:47,939 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3748fa113f244044ae3a0214ef3894bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=578, earliestPutTs=1733400945000 2024-12-05T12:15:47,940 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 32409f74d5ca40ea95389b1d0a5ee3b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1733400947055 2024-12-05T12:15:47,940 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting bea4c30c2327436da10cfb74b48b8d98, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1733400947055 2024-12-05T12:15:47,947 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:47,947 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#A#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:47,948 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/1a842d63386e420a89f5b2c48f552099 is 50, key is test_row_0/B:col10/1733400947084/Put/seqid=0 2024-12-05T12:15:47,948 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/ae6da46634f442d59f6986b6298376fe is 50, key is test_row_0/A:col10/1733400947084/Put/seqid=0 2024-12-05T12:15:47,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742328_1504 (size=13153) 2024-12-05T12:15:47,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742327_1503 (size=13153) 2024-12-05T12:15:48,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-05T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:48,031 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-05T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:48,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/b33e9e41c13c427f90b1fe576c22e28d is 50, key is test_row_0/A:col10/1733400947151/Put/seqid=0 2024-12-05T12:15:48,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742329_1505 (size=12301) 2024-12-05T12:15:48,039 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/b33e9e41c13c427f90b1fe576c22e28d 2024-12-05T12:15:48,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/7bde56c4924a48d896134bbd1756caa9 is 50, key is test_row_0/B:col10/1733400947151/Put/seqid=0 2024-12-05T12:15:48,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742330_1506 (size=12301) 2024-12-05T12:15:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T12:15:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:48,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. as already flushing 2024-12-05T12:15:48,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733401008341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:48,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733401008342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:48,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733401008342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:48,361 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/ae6da46634f442d59f6986b6298376fe as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ae6da46634f442d59f6986b6298376fe 2024-12-05T12:15:48,363 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/1a842d63386e420a89f5b2c48f552099 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/1a842d63386e420a89f5b2c48f552099 2024-12-05T12:15:48,365 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/A of 7a7e5a7bd5645c444a2a620ce27fbe6a into ae6da46634f442d59f6986b6298376fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:48,365 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:48,365 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/A, priority=13, startTime=1733400947938; duration=0sec 2024-12-05T12:15:48,365 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:15:48,365 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/B of 7a7e5a7bd5645c444a2a620ce27fbe6a into 1a842d63386e420a89f5b2c48f552099(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:48,365 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:A 2024-12-05T12:15:48,365 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:48,365 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/B, priority=13, startTime=1733400947938; duration=0sec 2024-12-05T12:15:48,365 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:15:48,365 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:48,366 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:B 2024-12-05T12:15:48,366 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:15:48,366 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 7a7e5a7bd5645c444a2a620ce27fbe6a/C is initiating minor compaction (all files) 2024-12-05T12:15:48,366 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a7e5a7bd5645c444a2a620ce27fbe6a/C in TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:48,366 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/6f602aa9c12c4cc5b7d161da5758ac88, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/3416558eed834e7bae20f8b28a5d886f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/30744fd21dff49e9821a185781835590] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp, totalSize=36.8 K 2024-12-05T12:15:48,367 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f602aa9c12c4cc5b7d161da5758ac88, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1733400944331 2024-12-05T12:15:48,367 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3416558eed834e7bae20f8b28a5d886f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=578, earliestPutTs=1733400945000 2024-12-05T12:15:48,367 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30744fd21dff49e9821a185781835590, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1733400947055 2024-12-05T12:15:48,373 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a7e5a7bd5645c444a2a620ce27fbe6a#C#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:15:48,373 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/0caf6ba1bfdd481a9bab2c9558be4ebc is 50, key is test_row_0/C:col10/1733400947084/Put/seqid=0 2024-12-05T12:15:48,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742331_1507 (size=13153) 2024-12-05T12:15:48,383 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/0caf6ba1bfdd481a9bab2c9558be4ebc as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/0caf6ba1bfdd481a9bab2c9558be4ebc 2024-12-05T12:15:48,387 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a7e5a7bd5645c444a2a620ce27fbe6a/C of 7a7e5a7bd5645c444a2a620ce27fbe6a into 0caf6ba1bfdd481a9bab2c9558be4ebc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:15:48,387 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:48,387 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., storeName=7a7e5a7bd5645c444a2a620ce27fbe6a/C, priority=13, startTime=1733400947938; duration=0sec 2024-12-05T12:15:48,387 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:48,387 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:C 2024-12-05T12:15:48,452 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/7bde56c4924a48d896134bbd1756caa9 2024-12-05T12:15:48,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733401008453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:48,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733401008453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:48,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/2248f20e49ec4ff38efcfd0e76e3d53b is 50, key is test_row_0/C:col10/1733400947151/Put/seqid=0 2024-12-05T12:15:48,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742332_1508 (size=12301) 2024-12-05T12:15:48,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733401008463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:48,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49466 deadline: 1733401008662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:48,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49440 deadline: 1733401008663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:48,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49396 deadline: 1733401008673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 , the old value is region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=2 because the exception is null or not the one we care about 2024-12-05T12:15:48,708 INFO [Thread-1942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1942. 2024-12-05T12:15:48,708 DEBUG [Thread-1942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:48,708 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,708 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:48,711 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-12-05T12:15:48,711 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:48,711 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,711 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:48,714 INFO [Thread-1946 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1946. 2024-12-05T12:15:48,714 DEBUG [Thread-1946 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:48,714 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,715 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:48,717 INFO [Thread-1940 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1940. 2024-12-05T12:15:48,717 DEBUG [Thread-1940 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:48,717 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,717 INFO [Thread-1944 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1944. 2024-12-05T12:15:48,717 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:48,718 DEBUG [Thread-1944 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:48,718 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,718 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:48,864 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/2248f20e49ec4ff38efcfd0e76e3d53b 2024-12-05T12:15:48,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/b33e9e41c13c427f90b1fe576c22e28d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/b33e9e41c13c427f90b1fe576c22e28d 2024-12-05T12:15:48,879 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/b33e9e41c13c427f90b1fe576c22e28d, entries=150, sequenceid=626, filesize=12.0 K 2024-12-05T12:15:48,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/7bde56c4924a48d896134bbd1756caa9 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/7bde56c4924a48d896134bbd1756caa9 2024-12-05T12:15:48,885 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/7bde56c4924a48d896134bbd1756caa9, entries=150, sequenceid=626, filesize=12.0 K 2024-12-05T12:15:48,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/2248f20e49ec4ff38efcfd0e76e3d53b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/2248f20e49ec4ff38efcfd0e76e3d53b 2024-12-05T12:15:48,890 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/2248f20e49ec4ff38efcfd0e76e3d53b, entries=150, sequenceid=626, filesize=12.0 K 2024-12-05T12:15:48,891 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 860ms, sequenceid=626, compaction requested=false 2024-12-05T12:15:48,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:48,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:48,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-05T12:15:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-05T12:15:48,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-05T12:15:48,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-12-05T12:15:48,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.7800 sec 2024-12-05T12:15:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:48,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T12:15:48,977 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-12-05T12:15:48,977 INFO [Thread-1929 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1929. 2024-12-05T12:15:48,977 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:48,977 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,977 DEBUG [Thread-1929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:48,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:48,977 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,977 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:48,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:48,977 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:48,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:48,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:48,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/dc602be535a74df6b15f35325475ac4f is 50, key is test_row_0/A:col10/1733400948974/Put/seqid=0 2024-12-05T12:15:48,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742333_1509 (size=12301) 2024-12-05T12:15:48,995 INFO [Thread-1933 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1933. 2024-12-05T12:15:48,995 DEBUG [Thread-1933 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:48,995 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,995 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:48,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:49,207 INFO [Thread-1935 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1935. 2024-12-05T12:15:49,207 DEBUG [Thread-1935 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:49,207 DEBUG [Thread-1935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:49,207 DEBUG [Thread-1935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:49,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:49,216 INFO [Thread-1931 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1931. 2024-12-05T12:15:49,216 DEBUG [Thread-1931 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:15:49,216 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:49,216 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:49,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T12:15:49,264 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 222 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 262 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1731 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5191 rows 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1728 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5183 rows 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1733 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5199 rows 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1737 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5209 rows 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1741 2024-12-05T12:15:49,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5221 rows 2024-12-05T12:15:49,264 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:15:49,264 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:15:49,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:49,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:49,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:15:49,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T12:15:49,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T12:15:49,267 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400949267"}]},"ts":"1733400949267"} 2024-12-05T12:15:49,268 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T12:15:49,268 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T12:15:49,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T12:15:49,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a7e5a7bd5645c444a2a620ce27fbe6a, UNASSIGN}] 2024-12-05T12:15:49,271 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a7e5a7bd5645c444a2a620ce27fbe6a, UNASSIGN 2024-12-05T12:15:49,272 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=7a7e5a7bd5645c444a2a620ce27fbe6a, regionState=CLOSING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:49,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a7e5a7bd5645c444a2a620ce27fbe6a, UNASSIGN because future has completed 2024-12-05T12:15:49,273 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T12:15:49,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:15:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T12:15:49,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/dc602be535a74df6b15f35325475ac4f 2024-12-05T12:15:49,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/12c5c4a75aab4f948b466d06e4051862 is 50, key is test_row_0/B:col10/1733400948974/Put/seqid=0 2024-12-05T12:15:49,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742334_1510 (size=12301) 2024-12-05T12:15:49,425 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:49,425 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T12:15:49,425 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 7a7e5a7bd5645c444a2a620ce27fbe6a, disabling compactions & flushes 2024-12-05T12:15:49,426 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T12:15:49,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/12c5c4a75aab4f948b466d06e4051862 2024-12-05T12:15:49,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/84908a343d3b42e6bbccb1c8e6178c97 is 50, key is test_row_0/C:col10/1733400948974/Put/seqid=0 2024-12-05T12:15:49,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742335_1511 (size=12301) 2024-12-05T12:15:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T12:15:50,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/84908a343d3b42e6bbccb1c8e6178c97 2024-12-05T12:15:50,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/dc602be535a74df6b15f35325475ac4f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/dc602be535a74df6b15f35325475ac4f 2024-12-05T12:15:50,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/dc602be535a74df6b15f35325475ac4f, entries=150, sequenceid=641, filesize=12.0 K 2024-12-05T12:15:50,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/12c5c4a75aab4f948b466d06e4051862 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/12c5c4a75aab4f948b466d06e4051862 2024-12-05T12:15:50,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/12c5c4a75aab4f948b466d06e4051862, entries=150, sequenceid=641, filesize=12.0 K 2024-12-05T12:15:50,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/84908a343d3b42e6bbccb1c8e6178c97 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/84908a343d3b42e6bbccb1c8e6178c97 2024-12-05T12:15:50,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/84908a343d3b42e6bbccb1c8e6178c97, entries=150, sequenceid=641, filesize=12.0 K 2024-12-05T12:15:50,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 1263ms, sequenceid=641, compaction requested=true 2024-12-05T12:15:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: 2024-12-05T12:15:50,240 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:50,240 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:15:50,240 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. after waiting 0 ms 2024-12-05T12:15:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:50,240 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. because compaction request was cancelled 2024-12-05T12:15:50,240 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:50,240 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:A 2024-12-05T12:15:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:50,240 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. because compaction request was cancelled 2024-12-05T12:15:50,240 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:B 2024-12-05T12:15:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a7e5a7bd5645c444a2a620ce27fbe6a:C, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:15:50,240 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-05T12:15:50,240 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. because compaction request was cancelled 2024-12-05T12:15:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:15:50,240 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a7e5a7bd5645c444a2a620ce27fbe6a:C 2024-12-05T12:15:50,240 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=A 2024-12-05T12:15:50,240 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:50,240 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=B 2024-12-05T12:15:50,240 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:50,240 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a7e5a7bd5645c444a2a620ce27fbe6a, store=C 2024-12-05T12:15:50,240 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:50,244 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/d348f3bd5b4c4ca59b93fe8b491574d0 is 50, key is test_row_0/A:col10/1733400949204/Put/seqid=0 2024-12-05T12:15:50,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742336_1512 (size=12301) 2024-12-05T12:15:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T12:15:50,649 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/d348f3bd5b4c4ca59b93fe8b491574d0 2024-12-05T12:15:50,657 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/ea4e5d1848284f1baf05fc5ce9d6ef62 is 50, key is test_row_0/B:col10/1733400949204/Put/seqid=0 2024-12-05T12:15:50,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742337_1513 (size=12301) 2024-12-05T12:15:51,064 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/ea4e5d1848284f1baf05fc5ce9d6ef62 2024-12-05T12:15:51,076 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/d02ec931a7ae4c6383e63d563e360640 is 50, key is test_row_0/C:col10/1733400949204/Put/seqid=0 2024-12-05T12:15:51,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742338_1514 (size=12301) 2024-12-05T12:15:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T12:15:51,482 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/d02ec931a7ae4c6383e63d563e360640 2024-12-05T12:15:51,492 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/A/d348f3bd5b4c4ca59b93fe8b491574d0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/d348f3bd5b4c4ca59b93fe8b491574d0 2024-12-05T12:15:51,499 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/d348f3bd5b4c4ca59b93fe8b491574d0, entries=150, sequenceid=647, filesize=12.0 K 2024-12-05T12:15:51,500 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/B/ea4e5d1848284f1baf05fc5ce9d6ef62 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ea4e5d1848284f1baf05fc5ce9d6ef62 2024-12-05T12:15:51,504 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ea4e5d1848284f1baf05fc5ce9d6ef62, entries=150, sequenceid=647, filesize=12.0 K 2024-12-05T12:15:51,504 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/.tmp/C/d02ec931a7ae4c6383e63d563e360640 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/d02ec931a7ae4c6383e63d563e360640 2024-12-05T12:15:51,508 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/d02ec931a7ae4c6383e63d563e360640, entries=150, sequenceid=647, filesize=12.0 K 2024-12-05T12:15:51,509 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 1269ms, sequenceid=647, compaction requested=true 2024-12-05T12:15:51,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/29c8433b022243e99086d7caf925804f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/f4f07f4afb424e11a4a9ffea61f65b36, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/eccbd4bc54824cfea03ebbf4ac67374e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/11beab3532e842d396a7a21531cc35a4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/b2bca433899843b58bda5dae44b32e0b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/c2ba0d4fef7b4c869e25f47d3dd133c2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/d43d6ed3ef1c43699219edd07eeefbae, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/163705d33a8941fc906730d55006ba6d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/071fd700dec8474d93f1bda26d0ab6f0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/7a31452edf92406e8edf8ab07f7b3bd8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e90f8ba5d2844d51952fdc9c1b905f8f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6a6edcfb71f5418d8c29929f6e2c5791, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2a2023e54daf44bbbdde29e315e6375b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/4646ac193fb74924a30736b4275ec39c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2adee0cfbde345e6beec32b344f609a2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/32f038ffc026492a88b80f2d9b87e98e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ce24aa11a53c429e82ad18e99c9d2c09, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3941dd157252437893e96b6b73ef1c49, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6d062abb8a9a4dbf944495d168114ee1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e87fe22811a145c2ba75c77fa57f6a35, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/032434045de9475eb8dac486319e5ae5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ccece7d347e84ce9ba1aa9f25a618630, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/42b3cb14155d4fc39b36360d3119852a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3748fa113f244044ae3a0214ef3894bf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/bea4c30c2327436da10cfb74b48b8d98] to archive 2024-12-05T12:15:51,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:15:51,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/29c8433b022243e99086d7caf925804f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/29c8433b022243e99086d7caf925804f 2024-12-05T12:15:51,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/f4f07f4afb424e11a4a9ffea61f65b36 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/f4f07f4afb424e11a4a9ffea61f65b36 2024-12-05T12:15:51,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/eccbd4bc54824cfea03ebbf4ac67374e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/eccbd4bc54824cfea03ebbf4ac67374e 2024-12-05T12:15:51,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/11beab3532e842d396a7a21531cc35a4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/11beab3532e842d396a7a21531cc35a4 2024-12-05T12:15:51,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/b2bca433899843b58bda5dae44b32e0b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/b2bca433899843b58bda5dae44b32e0b 2024-12-05T12:15:51,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/c2ba0d4fef7b4c869e25f47d3dd133c2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/c2ba0d4fef7b4c869e25f47d3dd133c2 2024-12-05T12:15:51,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/d43d6ed3ef1c43699219edd07eeefbae to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/d43d6ed3ef1c43699219edd07eeefbae 2024-12-05T12:15:51,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/163705d33a8941fc906730d55006ba6d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/163705d33a8941fc906730d55006ba6d 2024-12-05T12:15:51,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/071fd700dec8474d93f1bda26d0ab6f0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/071fd700dec8474d93f1bda26d0ab6f0 2024-12-05T12:15:51,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/7a31452edf92406e8edf8ab07f7b3bd8 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/7a31452edf92406e8edf8ab07f7b3bd8 2024-12-05T12:15:51,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e90f8ba5d2844d51952fdc9c1b905f8f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e90f8ba5d2844d51952fdc9c1b905f8f 2024-12-05T12:15:51,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6a6edcfb71f5418d8c29929f6e2c5791 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6a6edcfb71f5418d8c29929f6e2c5791 2024-12-05T12:15:51,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2a2023e54daf44bbbdde29e315e6375b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2a2023e54daf44bbbdde29e315e6375b 2024-12-05T12:15:51,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/4646ac193fb74924a30736b4275ec39c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/4646ac193fb74924a30736b4275ec39c 2024-12-05T12:15:51,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2adee0cfbde345e6beec32b344f609a2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/2adee0cfbde345e6beec32b344f609a2 2024-12-05T12:15:51,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/32f038ffc026492a88b80f2d9b87e98e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/32f038ffc026492a88b80f2d9b87e98e 2024-12-05T12:15:51,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ce24aa11a53c429e82ad18e99c9d2c09 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ce24aa11a53c429e82ad18e99c9d2c09 2024-12-05T12:15:51,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3941dd157252437893e96b6b73ef1c49 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3941dd157252437893e96b6b73ef1c49 2024-12-05T12:15:51,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6d062abb8a9a4dbf944495d168114ee1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/6d062abb8a9a4dbf944495d168114ee1 2024-12-05T12:15:51,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e87fe22811a145c2ba75c77fa57f6a35 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/e87fe22811a145c2ba75c77fa57f6a35 2024-12-05T12:15:51,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/032434045de9475eb8dac486319e5ae5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/032434045de9475eb8dac486319e5ae5 2024-12-05T12:15:51,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ccece7d347e84ce9ba1aa9f25a618630 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ccece7d347e84ce9ba1aa9f25a618630 2024-12-05T12:15:51,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/42b3cb14155d4fc39b36360d3119852a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/42b3cb14155d4fc39b36360d3119852a 2024-12-05T12:15:51,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3748fa113f244044ae3a0214ef3894bf to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/3748fa113f244044ae3a0214ef3894bf 2024-12-05T12:15:51,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/bea4c30c2327436da10cfb74b48b8d98 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/bea4c30c2327436da10cfb74b48b8d98 2024-12-05T12:15:51,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/20190dcc1f48483088d58448c83da3b3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/783040155d704e5f81cbbfc02b9d40d7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/33dd5f0c65c849b2873537a91d0abf84, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/7a10dbfd7a9d4523be8deaa8b773151c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/773b4dfafb96493ab0c96166f21c79b7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/01c3b9b3f30e4fb3bc409675e26f0353, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9fd2ae093434407ea425ea781ba2c9e1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/a2cdf3a4de0245398c9a179fd86c0c63, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9bd7ac543bcb444792be4f1c22e33695, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/c092ec43c55f4a8d98eb0fa39d0970c2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/17144d98860c4800841a551840dc8647, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/824423ccc1654c1187ba38e1e9649bc0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/648e5aa01f46446c8b6cc4acbf42b82b, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/b9ea047fef114951bbc7cf653a1bd387, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/546105c4c4ff44d188abe6ad82d0a63e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bd5ff75cf8a14e7484f0b5c98f30d5f3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bfcaffdcd0c746b29136bf320f2abda6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/0dd110ede7454ff884daeb19b2ee930a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4bfbf2d0596e4821aec6b9e0d22a7d43, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/e568f55b4ea64792b73bbe4e99e34f60, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab5fcec9a57045b38c3f33eea317f6af, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab6eb7979c12409e8f40d5ef8cb8dd20, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/d53909de1b07429986cb274a8a412197, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4557d3b6726a47ff8cf645e24776312c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/32409f74d5ca40ea95389b1d0a5ee3b3] to archive 2024-12-05T12:15:51,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:15:51,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/20190dcc1f48483088d58448c83da3b3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/20190dcc1f48483088d58448c83da3b3 2024-12-05T12:15:51,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/783040155d704e5f81cbbfc02b9d40d7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/783040155d704e5f81cbbfc02b9d40d7 2024-12-05T12:15:51,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/33dd5f0c65c849b2873537a91d0abf84 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/33dd5f0c65c849b2873537a91d0abf84 2024-12-05T12:15:51,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/7a10dbfd7a9d4523be8deaa8b773151c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/7a10dbfd7a9d4523be8deaa8b773151c 2024-12-05T12:15:51,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/773b4dfafb96493ab0c96166f21c79b7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/773b4dfafb96493ab0c96166f21c79b7 2024-12-05T12:15:51,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/01c3b9b3f30e4fb3bc409675e26f0353 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/01c3b9b3f30e4fb3bc409675e26f0353 2024-12-05T12:15:51,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9fd2ae093434407ea425ea781ba2c9e1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9fd2ae093434407ea425ea781ba2c9e1 2024-12-05T12:15:51,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/a2cdf3a4de0245398c9a179fd86c0c63 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/a2cdf3a4de0245398c9a179fd86c0c63 2024-12-05T12:15:51,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9bd7ac543bcb444792be4f1c22e33695 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/9bd7ac543bcb444792be4f1c22e33695 2024-12-05T12:15:51,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/c092ec43c55f4a8d98eb0fa39d0970c2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/c092ec43c55f4a8d98eb0fa39d0970c2 2024-12-05T12:15:51,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/17144d98860c4800841a551840dc8647 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/17144d98860c4800841a551840dc8647 2024-12-05T12:15:51,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/824423ccc1654c1187ba38e1e9649bc0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/824423ccc1654c1187ba38e1e9649bc0 2024-12-05T12:15:51,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/648e5aa01f46446c8b6cc4acbf42b82b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/648e5aa01f46446c8b6cc4acbf42b82b 2024-12-05T12:15:51,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/b9ea047fef114951bbc7cf653a1bd387 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/b9ea047fef114951bbc7cf653a1bd387 2024-12-05T12:15:51,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/546105c4c4ff44d188abe6ad82d0a63e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/546105c4c4ff44d188abe6ad82d0a63e 2024-12-05T12:15:51,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bd5ff75cf8a14e7484f0b5c98f30d5f3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bd5ff75cf8a14e7484f0b5c98f30d5f3 2024-12-05T12:15:51,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bfcaffdcd0c746b29136bf320f2abda6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/bfcaffdcd0c746b29136bf320f2abda6 2024-12-05T12:15:51,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/0dd110ede7454ff884daeb19b2ee930a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/0dd110ede7454ff884daeb19b2ee930a 2024-12-05T12:15:51,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4bfbf2d0596e4821aec6b9e0d22a7d43 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4bfbf2d0596e4821aec6b9e0d22a7d43 2024-12-05T12:15:51,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/e568f55b4ea64792b73bbe4e99e34f60 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/e568f55b4ea64792b73bbe4e99e34f60 2024-12-05T12:15:51,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab5fcec9a57045b38c3f33eea317f6af to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab5fcec9a57045b38c3f33eea317f6af 2024-12-05T12:15:51,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab6eb7979c12409e8f40d5ef8cb8dd20 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ab6eb7979c12409e8f40d5ef8cb8dd20 2024-12-05T12:15:51,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/d53909de1b07429986cb274a8a412197 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/d53909de1b07429986cb274a8a412197 2024-12-05T12:15:51,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4557d3b6726a47ff8cf645e24776312c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/4557d3b6726a47ff8cf645e24776312c 2024-12-05T12:15:51,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/32409f74d5ca40ea95389b1d0a5ee3b3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/32409f74d5ca40ea95389b1d0a5ee3b3 2024-12-05T12:15:51,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/fc35426e63f147049d5ba2aafaa22966, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7b5088f9b4bb446c982db20ebdd16e8f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/76720493133543e8a759c61ead1e76b3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/ffd2520d638d4ee7a9527365638276b5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b1fa00e4b4b04ba9ba462287537f5283, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7123cde97a204829a1906777462cd10a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/721b225d62a747b69d3db70e5d26d2cb, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5a13006acd3f4931a5cc33b56be5c5b1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5e2b73c62e904dc483c83b3376f7da61, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/c612f36abfbb417c911672c573409eca, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/e1dc3e955c1a45ba8b1883c160bf718c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/0a7336acbc644bf48582bab1c2bf3379, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7c8d8a9a3f77455e88e517baeab9ab36, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/106dc2fadb9c42ccbd501bdeb69ebc75, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b994213ea30b4055835133fd0d3158fc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/aec55dade5c14bbd831f335c9083f77f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b3a9e4e7d25e41c5963802f1c5a3b5f2, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/66096e2c03bd4d89bce75d08047a6049, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/77968d7bbd9f4a44ab8d8829c58a82be, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/08cf852574f04d32a650dd974bb9aad5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/acdf96d058e14845b099aa2131aa05a1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/6f602aa9c12c4cc5b7d161da5758ac88, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/05ace6d36029424bbdfe50f493a3bde6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/3416558eed834e7bae20f8b28a5d886f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/30744fd21dff49e9821a185781835590] to archive 2024-12-05T12:15:51,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:15:51,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/fc35426e63f147049d5ba2aafaa22966 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/fc35426e63f147049d5ba2aafaa22966 2024-12-05T12:15:51,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7b5088f9b4bb446c982db20ebdd16e8f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7b5088f9b4bb446c982db20ebdd16e8f 2024-12-05T12:15:51,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/76720493133543e8a759c61ead1e76b3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/76720493133543e8a759c61ead1e76b3 2024-12-05T12:15:51,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/ffd2520d638d4ee7a9527365638276b5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/ffd2520d638d4ee7a9527365638276b5 2024-12-05T12:15:51,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b1fa00e4b4b04ba9ba462287537f5283 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b1fa00e4b4b04ba9ba462287537f5283 2024-12-05T12:15:51,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7123cde97a204829a1906777462cd10a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7123cde97a204829a1906777462cd10a 2024-12-05T12:15:51,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/721b225d62a747b69d3db70e5d26d2cb to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/721b225d62a747b69d3db70e5d26d2cb 2024-12-05T12:15:51,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5a13006acd3f4931a5cc33b56be5c5b1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5a13006acd3f4931a5cc33b56be5c5b1 2024-12-05T12:15:51,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5e2b73c62e904dc483c83b3376f7da61 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/5e2b73c62e904dc483c83b3376f7da61 2024-12-05T12:15:51,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/c612f36abfbb417c911672c573409eca to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/c612f36abfbb417c911672c573409eca 2024-12-05T12:15:51,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/e1dc3e955c1a45ba8b1883c160bf718c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/e1dc3e955c1a45ba8b1883c160bf718c 2024-12-05T12:15:51,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/0a7336acbc644bf48582bab1c2bf3379 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/0a7336acbc644bf48582bab1c2bf3379 2024-12-05T12:15:51,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7c8d8a9a3f77455e88e517baeab9ab36 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/7c8d8a9a3f77455e88e517baeab9ab36 2024-12-05T12:15:51,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/106dc2fadb9c42ccbd501bdeb69ebc75 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/106dc2fadb9c42ccbd501bdeb69ebc75 2024-12-05T12:15:51,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b994213ea30b4055835133fd0d3158fc to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b994213ea30b4055835133fd0d3158fc 2024-12-05T12:15:51,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/aec55dade5c14bbd831f335c9083f77f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/aec55dade5c14bbd831f335c9083f77f 2024-12-05T12:15:51,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b3a9e4e7d25e41c5963802f1c5a3b5f2 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/b3a9e4e7d25e41c5963802f1c5a3b5f2 2024-12-05T12:15:51,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/66096e2c03bd4d89bce75d08047a6049 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/66096e2c03bd4d89bce75d08047a6049 2024-12-05T12:15:51,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/77968d7bbd9f4a44ab8d8829c58a82be to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/77968d7bbd9f4a44ab8d8829c58a82be 2024-12-05T12:15:51,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/08cf852574f04d32a650dd974bb9aad5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/08cf852574f04d32a650dd974bb9aad5 2024-12-05T12:15:51,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/acdf96d058e14845b099aa2131aa05a1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/acdf96d058e14845b099aa2131aa05a1 2024-12-05T12:15:51,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/6f602aa9c12c4cc5b7d161da5758ac88 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/6f602aa9c12c4cc5b7d161da5758ac88 2024-12-05T12:15:51,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/05ace6d36029424bbdfe50f493a3bde6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/05ace6d36029424bbdfe50f493a3bde6 2024-12-05T12:15:51,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/3416558eed834e7bae20f8b28a5d886f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/3416558eed834e7bae20f8b28a5d886f 2024-12-05T12:15:51,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/30744fd21dff49e9821a185781835590 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/30744fd21dff49e9821a185781835590 2024-12-05T12:15:51,588 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/recovered.edits/650.seqid, newMaxSeqId=650, maxSeqId=1 2024-12-05T12:15:51,588 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. 2024-12-05T12:15:51,588 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 7a7e5a7bd5645c444a2a620ce27fbe6a: Waiting for close lock at 1733400949425Running coprocessor pre-close hooks at 1733400949425Disabling compacts and flushes for region at 1733400949425Disabling writes for close at 1733400950240 (+815 ms)Obtaining lock to block concurrent updates at 1733400950240Preparing flush snapshotting stores in 7a7e5a7bd5645c444a2a620ce27fbe6a at 1733400950240Finished memstore snapshotting TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733400950241 (+1 ms)Flushing stores of TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a. at 1733400950241Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a/A: creating writer at 1733400950241Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a/A: appending metadata at 1733400950244 (+3 ms)Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a/A: closing flushed file at 1733400950244Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a/B: creating writer at 1733400950654 (+410 ms)Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a/B: appending metadata at 1733400950657 (+3 ms)Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a/B: closing flushed file at 1733400950657Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a/C: creating writer at 1733400951072 (+415 ms)Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a/C: appending metadata at 1733400951076 (+4 ms)Flushing 7a7e5a7bd5645c444a2a620ce27fbe6a/C: closing flushed file at 1733400951076Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b883710: reopening flushed file at 1733400951490 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75074c19: reopening flushed file at 1733400951500 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@323234a7: reopening flushed file at 1733400951504 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 7a7e5a7bd5645c444a2a620ce27fbe6a in 1269ms, sequenceid=647, compaction requested=true at 1733400951509 (+5 ms)Writing region close event to WAL at 1733400951585 (+76 ms)Running coprocessor post-close hooks at 1733400951588 (+3 ms)Closed at 1733400951588 2024-12-05T12:15:51,590 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:51,590 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=7a7e5a7bd5645c444a2a620ce27fbe6a, regionState=CLOSED 2024-12-05T12:15:51,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:15:51,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-05T12:15:51,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 7a7e5a7bd5645c444a2a620ce27fbe6a, server=9ea9abad0e9b,44759,1733400796278 in 2.3190 sec 2024-12-05T12:15:51,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-05T12:15:51,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a7e5a7bd5645c444a2a620ce27fbe6a, UNASSIGN in 2.3240 sec 2024-12-05T12:15:51,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-05T12:15:51,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3270 sec 2024-12-05T12:15:51,599 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400951599"}]},"ts":"1733400951599"} 2024-12-05T12:15:51,600 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T12:15:51,600 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T12:15:51,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3360 sec 2024-12-05T12:15:53,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T12:15:53,424 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:53,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T12:15:53,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:53,430 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-05T12:15:53,432 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:53,435 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:53,437 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/recovered.edits] 2024-12-05T12:15:53,440 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ae6da46634f442d59f6986b6298376fe to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/ae6da46634f442d59f6986b6298376fe 2024-12-05T12:15:53,442 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/b33e9e41c13c427f90b1fe576c22e28d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/b33e9e41c13c427f90b1fe576c22e28d 2024-12-05T12:15:53,443 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/d348f3bd5b4c4ca59b93fe8b491574d0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/d348f3bd5b4c4ca59b93fe8b491574d0 2024-12-05T12:15:53,445 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/dc602be535a74df6b15f35325475ac4f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/A/dc602be535a74df6b15f35325475ac4f 2024-12-05T12:15:53,448 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/12c5c4a75aab4f948b466d06e4051862 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/12c5c4a75aab4f948b466d06e4051862 2024-12-05T12:15:53,450 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/1a842d63386e420a89f5b2c48f552099 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/1a842d63386e420a89f5b2c48f552099 2024-12-05T12:15:53,452 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/7bde56c4924a48d896134bbd1756caa9 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/7bde56c4924a48d896134bbd1756caa9 2024-12-05T12:15:53,453 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ea4e5d1848284f1baf05fc5ce9d6ef62 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/B/ea4e5d1848284f1baf05fc5ce9d6ef62 2024-12-05T12:15:53,457 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/0caf6ba1bfdd481a9bab2c9558be4ebc to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/0caf6ba1bfdd481a9bab2c9558be4ebc 2024-12-05T12:15:53,458 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/2248f20e49ec4ff38efcfd0e76e3d53b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/2248f20e49ec4ff38efcfd0e76e3d53b 2024-12-05T12:15:53,460 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/84908a343d3b42e6bbccb1c8e6178c97 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/84908a343d3b42e6bbccb1c8e6178c97 2024-12-05T12:15:53,462 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/d02ec931a7ae4c6383e63d563e360640 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/C/d02ec931a7ae4c6383e63d563e360640 2024-12-05T12:15:53,466 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/recovered.edits/650.seqid to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a/recovered.edits/650.seqid 2024-12-05T12:15:53,466 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/7a7e5a7bd5645c444a2a620ce27fbe6a 2024-12-05T12:15:53,466 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T12:15:53,470 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:53,472 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T12:15:53,475 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T12:15:53,477 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:53,477 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T12:15:53,477 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733400953477"}]},"ts":"9223372036854775807"} 2024-12-05T12:15:53,479 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T12:15:53,479 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7a7e5a7bd5645c444a2a620ce27fbe6a, NAME => 'TestAcidGuarantees,,1733400926499.7a7e5a7bd5645c444a2a620ce27fbe6a.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T12:15:53,479 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T12:15:53,480 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733400953479"}]},"ts":"9223372036854775807"} 2024-12-05T12:15:53,481 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T12:15:53,482 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:53,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-12-05T12:15:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-05T12:15:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T12:15:53,544 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:53,556 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 252), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=467 (was 572), ProcessCount=11 (was 11), AvailableMemoryMB=3792 (was 3912) 2024-12-05T12:15:53,567 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=467, ProcessCount=11, AvailableMemoryMB=3791 2024-12-05T12:15:53,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T12:15:53,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:15:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:53,572 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T12:15:53,572 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:53,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-12-05T12:15:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T12:15:53,573 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T12:15:53,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742339_1515 (size=1061) 2024-12-05T12:15:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T12:15:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T12:15:53,984 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e 2024-12-05T12:15:54,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742340_1516 (size=53) 2024-12-05T12:15:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T12:15:54,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:15:54,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2e5f943f8689f12e595bd09f366346aa, disabling compactions & flushes 2024-12-05T12:15:54,404 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:54,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:54,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. after waiting 0 ms 2024-12-05T12:15:54,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:54,404 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:54,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2e5f943f8689f12e595bd09f366346aa: Waiting for close lock at 1733400954404Disabling compacts and flushes for region at 1733400954404Disabling writes for close at 1733400954404Writing region close event to WAL at 1733400954404Closed at 1733400954404 2024-12-05T12:15:54,407 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T12:15:54,407 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733400954407"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733400954407"}]},"ts":"1733400954407"} 2024-12-05T12:15:54,411 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T12:15:54,413 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T12:15:54,413 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400954413"}]},"ts":"1733400954413"} 2024-12-05T12:15:54,415 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T12:15:54,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, ASSIGN}] 2024-12-05T12:15:54,416 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, ASSIGN 2024-12-05T12:15:54,417 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, ASSIGN; state=OFFLINE, location=9ea9abad0e9b,44759,1733400796278; forceNewPlan=false, retain=false 2024-12-05T12:15:54,567 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=2e5f943f8689f12e595bd09f366346aa, regionState=OPENING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:54,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, ASSIGN because future has completed 2024-12-05T12:15:54,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:15:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T12:15:54,728 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:54,728 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:15:54,729 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,729 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:15:54,730 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,730 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,731 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,732 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:54,732 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e5f943f8689f12e595bd09f366346aa columnFamilyName A 2024-12-05T12:15:54,732 DEBUG [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:54,733 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(327): Store=2e5f943f8689f12e595bd09f366346aa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:54,733 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,734 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:54,734 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e5f943f8689f12e595bd09f366346aa columnFamilyName B 2024-12-05T12:15:54,734 DEBUG [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:54,734 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(327): Store=2e5f943f8689f12e595bd09f366346aa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:54,734 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,735 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:54,735 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e5f943f8689f12e595bd09f366346aa columnFamilyName C 2024-12-05T12:15:54,736 DEBUG [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:54,736 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(327): Store=2e5f943f8689f12e595bd09f366346aa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:54,736 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:54,736 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,737 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,737 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,738 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,738 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,738 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:15:54,739 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,741 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:15:54,741 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 2e5f943f8689f12e595bd09f366346aa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65045496, jitterRate=-0.03074657917022705}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:15:54,741 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:54,741 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 2e5f943f8689f12e595bd09f366346aa: Running coprocessor pre-open hook at 1733400954730Writing region info on filesystem at 1733400954730Initializing all the Stores at 1733400954731 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400954731Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400954731Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400954731Cleaning up temporary data from old regions at 1733400954738 (+7 ms)Running coprocessor post-open hooks at 1733400954741 (+3 ms)Region opened successfully at 1733400954741 2024-12-05T12:15:54,742 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., pid=144, masterSystemTime=1733400954722 2024-12-05T12:15:54,744 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:54,744 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:54,744 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=2e5f943f8689f12e595bd09f366346aa, regionState=OPEN, openSeqNum=2, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:54,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:15:54,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-05T12:15:54,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 in 177 msec 2024-12-05T12:15:54,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-05T12:15:54,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, ASSIGN in 332 msec 2024-12-05T12:15:54,749 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T12:15:54,749 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400954749"}]},"ts":"1733400954749"} 2024-12-05T12:15:54,750 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T12:15:54,751 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T12:15:54,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-12-05T12:15:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T12:15:55,724 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:55,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1847540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:55,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:55,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:55,729 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@544a01ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:55,732 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:55,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26246f45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:55,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:55,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:55,736 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T12:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ea9abad0e9b,34103,1733400795471 2024-12-05T12:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58f36db6 2024-12-05T12:15:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T12:15:55,738 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T12:15:55,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T12:15:55,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:15:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T12:15:55,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742341_1517 (size=1097) 2024-12-05T12:15:56,154 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T12:15:56,154 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T12:15:56,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T12:15:56,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, REOPEN/MOVE}] 2024-12-05T12:15:56,160 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, REOPEN/MOVE 2024-12-05T12:15:56,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=2e5f943f8689f12e595bd09f366346aa, regionState=CLOSING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, REOPEN/MOVE because future has completed 2024-12-05T12:15:56,162 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T12:15:56,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:15:56,316 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,316 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T12:15:56,316 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 2e5f943f8689f12e595bd09f366346aa, disabling compactions & flushes 2024-12-05T12:15:56,316 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:56,316 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:56,316 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. after waiting 0 ms 2024-12-05T12:15:56,317 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:56,322 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T12:15:56,323 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:56,323 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 2e5f943f8689f12e595bd09f366346aa: Waiting for close lock at 1733400956316Running coprocessor pre-close hooks at 1733400956316Disabling compacts and flushes for region at 1733400956316Disabling writes for close at 1733400956316Writing region close event to WAL at 1733400956318 (+2 ms)Running coprocessor post-close hooks at 1733400956323 (+5 ms)Closed at 1733400956323 2024-12-05T12:15:56,324 WARN [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 2e5f943f8689f12e595bd09f366346aa to self. 2024-12-05T12:15:56,326 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,327 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=2e5f943f8689f12e595bd09f366346aa, regionState=CLOSED 2024-12-05T12:15:56,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:15:56,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-05T12:15:56,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 in 168 msec 2024-12-05T12:15:56,332 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, REOPEN/MOVE; state=CLOSED, location=9ea9abad0e9b,44759,1733400796278; forceNewPlan=false, retain=true 2024-12-05T12:15:56,483 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=2e5f943f8689f12e595bd09f366346aa, regionState=OPENING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, REOPEN/MOVE because future has completed 2024-12-05T12:15:56,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:15:56,639 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:56,640 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:15:56,640 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,640 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:15:56,640 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,640 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,642 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,642 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:56,643 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e5f943f8689f12e595bd09f366346aa columnFamilyName A 2024-12-05T12:15:56,644 DEBUG [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:56,644 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(327): Store=2e5f943f8689f12e595bd09f366346aa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:56,645 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,645 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:56,646 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e5f943f8689f12e595bd09f366346aa columnFamilyName B 2024-12-05T12:15:56,646 DEBUG [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:56,646 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(327): Store=2e5f943f8689f12e595bd09f366346aa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:56,646 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,647 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T12:15:56,647 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e5f943f8689f12e595bd09f366346aa columnFamilyName C 2024-12-05T12:15:56,647 DEBUG [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:56,647 INFO [StoreOpener-2e5f943f8689f12e595bd09f366346aa-1 {}] regionserver.HStore(327): Store=2e5f943f8689f12e595bd09f366346aa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:15:56,647 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:56,647 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,648 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,649 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,650 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,650 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,650 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T12:15:56,651 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,652 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 2e5f943f8689f12e595bd09f366346aa; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66838284, jitterRate=-0.004031956195831299}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T12:15:56,652 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,653 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 2e5f943f8689f12e595bd09f366346aa: Running coprocessor pre-open hook at 1733400956640Writing region info on filesystem at 1733400956640Initializing all the Stores at 1733400956641 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400956641Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400956641Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400956642 (+1 ms)Cleaning up temporary data from old regions at 1733400956650 (+8 ms)Running coprocessor post-open hooks at 1733400956652 (+2 ms)Region opened successfully at 1733400956653 (+1 ms) 2024-12-05T12:15:56,654 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., pid=149, masterSystemTime=1733400956636 2024-12-05T12:15:56,655 DEBUG [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:56,655 INFO [RS_OPEN_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:56,656 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=2e5f943f8689f12e595bd09f366346aa, regionState=OPEN, openSeqNum=5, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:15:56,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-12-05T12:15:56,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 in 174 msec 2024-12-05T12:15:56,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-05T12:15:56,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, REOPEN/MOVE in 500 msec 2024-12-05T12:15:56,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-05T12:15:56,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-12-05T12:15:56,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 923 msec 2024-12-05T12:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-05T12:15:56,664 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:15:56,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e6069e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:56,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:56,666 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a575f6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:56,667 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629f2677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604e524e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:56,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:56,668 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:56,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:56,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:56,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86cd29d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:56,669 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c3f6e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f36d46f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:56,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:56,671 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ea5e34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:56,672 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:56,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cf1680, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5e61df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:56,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:56,673 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8ff6a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:56,674 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c396c1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4885acdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:56,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:56,676 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7807c890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:56,677 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6410d1e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@473abea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:56,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:56,678 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148334ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:56,679 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d429c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3420b4f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:56,680 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:56,681 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d9562f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:56,682 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435726a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1b2f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:56,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:56,684 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:56,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:56,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:56,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739f3365, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:56,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:56,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:56,685 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0071ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1102b71d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:56,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:56,686 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f2f5394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:56,688 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391fb80f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ff4806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ea9abad0e9b,34103,-1 for getting cluster id 2024-12-05T12:15:56,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:15:56,689 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb9d9d-e54c-41a0-a632-14306d0e2ea2' 2024-12-05T12:15:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:15:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb9d9d-e54c-41a0-a632-14306d0e2ea2" 2024-12-05T12:15:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d61dde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ea9abad0e9b,34103,-1] 2024-12-05T12:15:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:15:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:15:56,690 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:15:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739e4ebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:15:56,691 DEBUG [Thread-2320 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:56,691 DEBUG [Thread-2335 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:56,691 DEBUG [Thread-2331 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:56,692 DEBUG [Thread-2333 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:56,692 DEBUG [Thread-2318 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:56,692 DEBUG [Thread-2316 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:56,693 DEBUG [Thread-2324 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:56,693 DEBUG [Thread-2322 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:56,693 DEBUG [Thread-2329 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:56,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:56,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:56,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:56,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:56,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:56,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:15:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:56,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:56,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:56,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-05T12:15:56,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:56,698 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:15:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T12:15:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:56,698 DEBUG [Thread-2327 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:15:56,699 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:15:56,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:15:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ea9abad0e9b,44759,1733400796278, seqNum=-1] 2024-12-05T12:15:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:15:56,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:15:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5] 2024-12-05T12:15:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:56,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T12:15:56,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:15:56,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:56,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:15:56,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:56,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:15:56,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:56,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ec4e368bac704e0b8e423dfbeb02f833_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400956700/Put/seqid=0 2024-12-05T12:15:56,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401016744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401016745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:56,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401016747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400956734, max timestamp=1733400956747Num uniques 859; 2024-12-05T12:15:56,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:56,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400956734, max timestamp=1733400956747Num uniques 859; 2024-12-05T12:15:56,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:56,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400956734, max timestamp=1733400956747Num uniques 859; 2024-12-05T12:15:56,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:15:56,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742342_1518 (size=12154) 2024-12-05T12:15:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:56,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401016775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:56,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401016777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T12:15:56,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T12:15:56,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:56,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:15:56,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:56,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:56,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:56,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401016852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:56,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401016852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:56,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:56,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401016862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:56,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401016883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401016883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T12:15:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:15:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T12:15:57,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401017062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401017062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401017073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401017093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401017093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,105 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T12:15:57,152 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:57,155 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ec4e368bac704e0b8e423dfbeb02f833_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ec4e368bac704e0b8e423dfbeb02f833_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:57,156 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/afc25a716e664a9cbca24777b2bc38cf, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:15:57,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/afc25a716e664a9cbca24777b2bc38cf is 175, key is test_row_0/A:col10/1733400956700/Put/seqid=0 2024-12-05T12:15:57,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T12:15:57,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:15:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742343_1519 (size=30955) 2024-12-05T12:15:57,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T12:15:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:15:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T12:15:57,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401017372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401017372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401017383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401017403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401017403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T12:15:57,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:15:57,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,560 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/afc25a716e664a9cbca24777b2bc38cf 2024-12-05T12:15:57,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/916e6fc4872d44ca969a760dc7c86c46 is 50, key is test_row_0/B:col10/1733400956700/Put/seqid=0 2024-12-05T12:15:57,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742344_1520 (size=12001) 2024-12-05T12:15:57,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/916e6fc4872d44ca969a760dc7c86c46 2024-12-05T12:15:57,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/1a0248970862480fb0dc7a01c6f778b8 is 50, key is test_row_0/C:col10/1733400956700/Put/seqid=0 2024-12-05T12:15:57,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742345_1521 (size=12001) 2024-12-05T12:15:57,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T12:15:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:15:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T12:15:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:15:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T12:15:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401017883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401017883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401017892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401017913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401017913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:57,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T12:15:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:15:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:57,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:15:58,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/1a0248970862480fb0dc7a01c6f778b8 2024-12-05T12:15:58,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/afc25a716e664a9cbca24777b2bc38cf as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/afc25a716e664a9cbca24777b2bc38cf 2024-12-05T12:15:58,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/afc25a716e664a9cbca24777b2bc38cf, entries=150, sequenceid=16, filesize=30.2 K 2024-12-05T12:15:58,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/916e6fc4872d44ca969a760dc7c86c46 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/916e6fc4872d44ca969a760dc7c86c46 2024-12-05T12:15:58,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/916e6fc4872d44ca969a760dc7c86c46, entries=150, sequenceid=16, filesize=11.7 K 2024-12-05T12:15:58,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/1a0248970862480fb0dc7a01c6f778b8 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1a0248970862480fb0dc7a01c6f778b8 2024-12-05T12:15:58,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1a0248970862480fb0dc7a01c6f778b8, entries=150, sequenceid=16, filesize=11.7 K 2024-12-05T12:15:58,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 2e5f943f8689f12e595bd09f366346aa in 1299ms, sequenceid=16, compaction requested=false 2024-12-05T12:15:58,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:15:58,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T12:15:58,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:58,074 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-12-05T12:15:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:15:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:15:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:15:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:15:58,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205876b078798454cf5836bdb0bb564fb38_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400956776/Put/seqid=0 2024-12-05T12:15:58,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742346_1522 (size=12154) 2024-12-05T12:15:58,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:58,086 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205876b078798454cf5836bdb0bb564fb38_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205876b078798454cf5836bdb0bb564fb38_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:58,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/b14b95406530422e8da6244ba1cea20e, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:15:58,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/b14b95406530422e8da6244ba1cea20e is 175, key is test_row_0/A:col10/1733400956776/Put/seqid=0 2024-12-05T12:15:58,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742347_1523 (size=30955) 2024-12-05T12:15:58,491 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/b14b95406530422e8da6244ba1cea20e 2024-12-05T12:15:58,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/0b45c1a4d8454610a6c94ced1665aed1 is 50, key is test_row_0/B:col10/1733400956776/Put/seqid=0 2024-12-05T12:15:58,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742348_1524 (size=12001) 2024-12-05T12:15:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T12:15:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:58,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:15:58,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401018906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:58,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401018906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:58,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401018907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:58,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401018922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:58,925 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:58,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401018933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:58,935 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:15:58,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:58,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:58,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:58,944 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/0b45c1a4d8454610a6c94ced1665aed1 2024-12-05T12:15:58,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/aab05b6657794599be46411ec01cf248 is 50, key is test_row_0/C:col10/1733400956776/Put/seqid=0 2024-12-05T12:15:58,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742349_1525 (size=12001) 2024-12-05T12:15:58,961 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/aab05b6657794599be46411ec01cf248 2024-12-05T12:15:58,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/b14b95406530422e8da6244ba1cea20e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b14b95406530422e8da6244ba1cea20e 2024-12-05T12:15:58,967 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b14b95406530422e8da6244ba1cea20e, entries=150, sequenceid=51, filesize=30.2 K 2024-12-05T12:15:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/0b45c1a4d8454610a6c94ced1665aed1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/0b45c1a4d8454610a6c94ced1665aed1 2024-12-05T12:15:58,970 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/0b45c1a4d8454610a6c94ced1665aed1, entries=150, sequenceid=51, filesize=11.7 K 2024-12-05T12:15:58,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/aab05b6657794599be46411ec01cf248 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/aab05b6657794599be46411ec01cf248 2024-12-05T12:15:58,974 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/aab05b6657794599be46411ec01cf248, entries=150, sequenceid=51, filesize=11.7 K 2024-12-05T12:15:58,974 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 2e5f943f8689f12e595bd09f366346aa in 900ms, sequenceid=51, compaction requested=false 2024-12-05T12:15:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:15:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:15:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-05T12:15:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-05T12:15:58,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-05T12:15:58,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2770 sec 2024-12-05T12:15:58,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.2840 sec 2024-12-05T12:15:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:59,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T12:15:59,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:15:59,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:59,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:15:59,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:59,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:15:59,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:15:59,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120510d9cf71d7704b2ea2a4af20503c6cc5_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400958901/Put/seqid=0 2024-12-05T12:15:59,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742350_1526 (size=14594) 2024-12-05T12:15:59,025 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:15:59,028 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120510d9cf71d7704b2ea2a4af20503c6cc5_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120510d9cf71d7704b2ea2a4af20503c6cc5_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:15:59,029 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6bc80f301f804bee8b96d7b69286d79d, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:15:59,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6bc80f301f804bee8b96d7b69286d79d is 175, key is test_row_0/A:col10/1733400958901/Put/seqid=0 2024-12-05T12:15:59,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742351_1527 (size=39549) 2024-12-05T12:15:59,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401019039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401019040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401019041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401019153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401019153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401019153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401019362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401019362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401019363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6bc80f301f804bee8b96d7b69286d79d 2024-12-05T12:15:59,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/b4c5ac81e6f44c8c9616c10fe3bb8a78 is 50, key is test_row_0/B:col10/1733400958901/Put/seqid=0 2024-12-05T12:15:59,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742352_1528 (size=12001) 2024-12-05T12:15:59,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401019673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401019673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:15:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401019673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:15:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:15:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:15:59,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/b4c5ac81e6f44c8c9616c10fe3bb8a78 2024-12-05T12:15:59,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/88d528baaab64265a2effef0546f7381 is 50, key is test_row_0/C:col10/1733400958901/Put/seqid=0 2024-12-05T12:15:59,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742353_1529 (size=12001) 2024-12-05T12:16:00,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401020183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:00,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401020183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:00,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401020183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:00,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/88d528baaab64265a2effef0546f7381 2024-12-05T12:16:00,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6bc80f301f804bee8b96d7b69286d79d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6bc80f301f804bee8b96d7b69286d79d 2024-12-05T12:16:00,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6bc80f301f804bee8b96d7b69286d79d, entries=200, sequenceid=64, filesize=38.6 K 2024-12-05T12:16:00,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/b4c5ac81e6f44c8c9616c10fe3bb8a78 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/b4c5ac81e6f44c8c9616c10fe3bb8a78 2024-12-05T12:16:00,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/b4c5ac81e6f44c8c9616c10fe3bb8a78, entries=150, sequenceid=64, filesize=11.7 K 2024-12-05T12:16:00,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/88d528baaab64265a2effef0546f7381 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/88d528baaab64265a2effef0546f7381 2024-12-05T12:16:00,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/88d528baaab64265a2effef0546f7381, entries=150, sequenceid=64, filesize=11.7 K 2024-12-05T12:16:00,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 2e5f943f8689f12e595bd09f366346aa in 1247ms, sequenceid=64, compaction requested=true 2024-12-05T12:16:00,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:00,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:16:00,262 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:16:00,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:00,262 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:16:00,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:16:00,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:00,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:16:00,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:16:00,263 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:16:00,263 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:16:00,263 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/A is initiating minor compaction (all files) 2024-12-05T12:16:00,263 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/B is initiating minor compaction (all files) 2024-12-05T12:16:00,263 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/A in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:00,263 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/B in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:00,263 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/916e6fc4872d44ca969a760dc7c86c46, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/0b45c1a4d8454610a6c94ced1665aed1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/b4c5ac81e6f44c8c9616c10fe3bb8a78] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=35.2 K 2024-12-05T12:16:00,263 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/afc25a716e664a9cbca24777b2bc38cf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b14b95406530422e8da6244ba1cea20e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6bc80f301f804bee8b96d7b69286d79d] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=99.1 K 2024-12-05T12:16:00,263 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:00,263 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/afc25a716e664a9cbca24777b2bc38cf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b14b95406530422e8da6244ba1cea20e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6bc80f301f804bee8b96d7b69286d79d] 2024-12-05T12:16:00,264 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting afc25a716e664a9cbca24777b2bc38cf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733400956700 2024-12-05T12:16:00,264 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 916e6fc4872d44ca969a760dc7c86c46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733400956700 2024-12-05T12:16:00,264 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b45c1a4d8454610a6c94ced1665aed1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733400956769 2024-12-05T12:16:00,264 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b14b95406530422e8da6244ba1cea20e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733400956769 2024-12-05T12:16:00,264 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting b4c5ac81e6f44c8c9616c10fe3bb8a78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733400958901 2024-12-05T12:16:00,264 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bc80f301f804bee8b96d7b69286d79d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733400958901 2024-12-05T12:16:00,268 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:00,269 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:00,270 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/7329446d75c544199261f6f1bb0e54e6 is 50, key is test_row_0/B:col10/1733400958901/Put/seqid=0 2024-12-05T12:16:00,270 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205c1300375b0fe4ffaa4c15d9135d8b915_2e5f943f8689f12e595bd09f366346aa store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:00,272 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205c1300375b0fe4ffaa4c15d9135d8b915_2e5f943f8689f12e595bd09f366346aa, store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:00,272 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c1300375b0fe4ffaa4c15d9135d8b915_2e5f943f8689f12e595bd09f366346aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:00,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742354_1530 (size=12104) 2024-12-05T12:16:00,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742355_1531 (size=4469) 2024-12-05T12:16:00,300 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#A#compaction#451 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:00,301 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/14fe5035857345e99fcdc702e4d6f2ba is 175, key is test_row_0/A:col10/1733400958901/Put/seqid=0 2024-12-05T12:16:00,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742356_1532 (size=31058) 2024-12-05T12:16:00,307 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/14fe5035857345e99fcdc702e4d6f2ba as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/14fe5035857345e99fcdc702e4d6f2ba 2024-12-05T12:16:00,310 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/A of 2e5f943f8689f12e595bd09f366346aa into 14fe5035857345e99fcdc702e4d6f2ba(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:00,310 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:00,310 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/A, priority=13, startTime=1733400960262; duration=0sec 2024-12-05T12:16:00,311 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:16:00,311 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:A 2024-12-05T12:16:00,311 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:16:00,312 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:16:00,312 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/C is initiating minor compaction (all files) 2024-12-05T12:16:00,312 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/C in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:00,312 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1a0248970862480fb0dc7a01c6f778b8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/aab05b6657794599be46411ec01cf248, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/88d528baaab64265a2effef0546f7381] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=35.2 K 2024-12-05T12:16:00,312 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a0248970862480fb0dc7a01c6f778b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733400956700 2024-12-05T12:16:00,312 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting aab05b6657794599be46411ec01cf248, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733400956769 2024-12-05T12:16:00,312 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d528baaab64265a2effef0546f7381, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733400958901 2024-12-05T12:16:00,319 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#C#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:00,319 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/695128a333e041378caaa3e5f68b4ae1 is 50, key is test_row_0/C:col10/1733400958901/Put/seqid=0 2024-12-05T12:16:00,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742357_1533 (size=12104) 2024-12-05T12:16:00,702 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/7329446d75c544199261f6f1bb0e54e6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7329446d75c544199261f6f1bb0e54e6 2024-12-05T12:16:00,705 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/B of 2e5f943f8689f12e595bd09f366346aa into 7329446d75c544199261f6f1bb0e54e6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:00,705 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:00,705 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/B, priority=13, startTime=1733400960262; duration=0sec 2024-12-05T12:16:00,705 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:00,705 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:B 2024-12-05T12:16:00,726 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/695128a333e041378caaa3e5f68b4ae1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/695128a333e041378caaa3e5f68b4ae1 2024-12-05T12:16:00,729 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/C of 2e5f943f8689f12e595bd09f366346aa into 695128a333e041378caaa3e5f68b4ae1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:00,729 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:00,729 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/C, priority=13, startTime=1733400960263; duration=0sec 2024-12-05T12:16:00,729 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:00,729 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:C 2024-12-05T12:16:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T12:16:00,853 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:16:00,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:16:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-05T12:16:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T12:16:00,855 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:16:00,855 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:16:00,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:16:00,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T12:16:00,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:00,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:00,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:00,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:00,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:00,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:00,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c707e9c5641e484c8b7d733bd5685b1e_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400960943/Put/seqid=0 2024-12-05T12:16:00,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742358_1534 (size=12154) 2024-12-05T12:16:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T12:16:00,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401020991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:00,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401020995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:01,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401021103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:01,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:01,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401021113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:01,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T12:16:01,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401021193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:01,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401021193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:01,196 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:01,196 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:01,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401021202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:01,204 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:01,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401021312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:01,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:01,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:01,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:01,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401021322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:01,356 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:01,359 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c707e9c5641e484c8b7d733bd5685b1e_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c707e9c5641e484c8b7d733bd5685b1e_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:01,360 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/19e8681ab25d42b594108edcfc14493c, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:01,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/19e8681ab25d42b594108edcfc14493c is 175, key is test_row_0/A:col10/1733400960943/Put/seqid=0 2024-12-05T12:16:01,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742359_1535 (size=30955) 2024-12-05T12:16:01,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:01,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T12:16:01,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401021622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:01,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401021633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:01,763 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/19e8681ab25d42b594108edcfc14493c 2024-12-05T12:16:01,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/08dddc2cf5834ffeb2c0ee7373cc38c6 is 50, key is test_row_0/B:col10/1733400960943/Put/seqid=0 2024-12-05T12:16:01,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742360_1536 (size=12001) 2024-12-05T12:16:01,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:01,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:01,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:01,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:01,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T12:16:02,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:02,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:02,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:02,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:02,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401022132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:02,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:02,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:02,142 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T12:16:02,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401022143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:02,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/08dddc2cf5834ffeb2c0ee7373cc38c6 2024-12-05T12:16:02,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/62b6abcf4f714e3783e953e3ca96e06e is 50, key is test_row_0/C:col10/1733400960943/Put/seqid=0 2024-12-05T12:16:02,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742361_1537 (size=12001) 2024-12-05T12:16:02,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:02,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:02,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:02,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:02,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:02,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:02,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:02,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:02,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/62b6abcf4f714e3783e953e3ca96e06e 2024-12-05T12:16:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/19e8681ab25d42b594108edcfc14493c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/19e8681ab25d42b594108edcfc14493c 2024-12-05T12:16:02,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/19e8681ab25d42b594108edcfc14493c, entries=150, sequenceid=88, filesize=30.2 K 2024-12-05T12:16:02,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/08dddc2cf5834ffeb2c0ee7373cc38c6 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/08dddc2cf5834ffeb2c0ee7373cc38c6 2024-12-05T12:16:02,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/08dddc2cf5834ffeb2c0ee7373cc38c6, entries=150, sequenceid=88, filesize=11.7 K 2024-12-05T12:16:02,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/62b6abcf4f714e3783e953e3ca96e06e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/62b6abcf4f714e3783e953e3ca96e06e 2024-12-05T12:16:02,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/62b6abcf4f714e3783e953e3ca96e06e, entries=150, sequenceid=88, filesize=11.7 K 2024-12-05T12:16:02,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 2e5f943f8689f12e595bd09f366346aa in 1648ms, sequenceid=88, compaction requested=false 2024-12-05T12:16:02,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:02,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T12:16:02,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:02,692 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T12:16:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053c889d43598b4e969444c1e43b788b18_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400960993/Put/seqid=0 2024-12-05T12:16:02,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742362_1538 (size=12154) 2024-12-05T12:16:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T12:16:03,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:03,105 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053c889d43598b4e969444c1e43b788b18_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053c889d43598b4e969444c1e43b788b18_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/4a6a808219a14542910a50490790b2e1, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:03,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/4a6a808219a14542910a50490790b2e1 is 175, key is test_row_0/A:col10/1733400960993/Put/seqid=0 2024-12-05T12:16:03,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742363_1539 (size=30955) 2024-12-05T12:16:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:03,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:03,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400963170, max timestamp=1733400963177Num uniques 859; 2024-12-05T12:16:03,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:16:03,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400963170, max timestamp=1733400963177Num uniques 859; 2024-12-05T12:16:03,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:16:03,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733400963170, max timestamp=1733400963177Num uniques 859; 2024-12-05T12:16:03,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T12:16:03,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401023203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401023207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401023213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401023213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,214 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:03,215 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_1' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401023213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,216 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401023313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401023313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401023523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401023523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,524 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/4a6a808219a14542910a50490790b2e1 2024-12-05T12:16:03,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/9a997c9fc06e46dbb375b576d055dcc4 is 50, key is test_row_0/B:col10/1733400960993/Put/seqid=0 2024-12-05T12:16:03,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742364_1540 (size=12001) 2024-12-05T12:16:03,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401023832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401023832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:03,934 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/9a997c9fc06e46dbb375b576d055dcc4 2024-12-05T12:16:03,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/7cb73cef679c4b8c890965784fadc108 is 50, key is test_row_0/C:col10/1733400960993/Put/seqid=0 2024-12-05T12:16:03,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742365_1541 (size=12001) 2024-12-05T12:16:04,342 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/7cb73cef679c4b8c890965784fadc108 2024-12-05T12:16:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/4a6a808219a14542910a50490790b2e1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/4a6a808219a14542910a50490790b2e1 2024-12-05T12:16:04,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:04,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401024343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401024343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:04,348 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/4a6a808219a14542910a50490790b2e1, entries=150, sequenceid=99, filesize=30.2 K 2024-12-05T12:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:04,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/9a997c9fc06e46dbb375b576d055dcc4 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9a997c9fc06e46dbb375b576d055dcc4 2024-12-05T12:16:04,351 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9a997c9fc06e46dbb375b576d055dcc4, entries=150, sequenceid=99, filesize=11.7 K 2024-12-05T12:16:04,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/7cb73cef679c4b8c890965784fadc108 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/7cb73cef679c4b8c890965784fadc108 2024-12-05T12:16:04,354 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/7cb73cef679c4b8c890965784fadc108, entries=150, sequenceid=99, filesize=11.7 K 2024-12-05T12:16:04,355 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2e5f943f8689f12e595bd09f366346aa in 1663ms, sequenceid=99, compaction requested=true 2024-12-05T12:16:04,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:04,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:04,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-05T12:16:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-05T12:16:04,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-05T12:16:04,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5000 sec 2024-12-05T12:16:04,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.5060 sec 2024-12-05T12:16:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T12:16:05,023 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:16:05,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:16:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-05T12:16:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T12:16:05,025 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:16:05,026 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:16:05,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:16:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T12:16:05,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T12:16:05,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:05,178 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-05T12:16:05,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:05,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:05,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:05,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:05,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:05,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:05,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d241850caa0846348c1afd17c0d6f2b0_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400963201/Put/seqid=0 2024-12-05T12:16:05,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742366_1542 (size=12304) 2024-12-05T12:16:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T12:16:05,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:05,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401025374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:05,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401025376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:05,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401025483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:05,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401025483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:05,591 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d241850caa0846348c1afd17c0d6f2b0_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d241850caa0846348c1afd17c0d6f2b0_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:05,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/d4f26f134207454fa2aeaea4fe26f507, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:05,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/d4f26f134207454fa2aeaea4fe26f507 is 175, key is test_row_0/A:col10/1733400963201/Put/seqid=0 2024-12-05T12:16:05,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742367_1543 (size=31105) 2024-12-05T12:16:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T12:16:05,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401025693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:05,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401025693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:05,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:05,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:05,995 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/d4f26f134207454fa2aeaea4fe26f507 2024-12-05T12:16:06,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401026002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:06,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401026003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:06,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/7d8e3d714c1a4588b8b8470579044a35 is 50, key is test_row_0/B:col10/1733400963201/Put/seqid=0 2024-12-05T12:16:06,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742368_1544 (size=12151) 2024-12-05T12:16:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T12:16:06,429 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/7d8e3d714c1a4588b8b8470579044a35 2024-12-05T12:16:06,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/16662a1a077347d2a0cadcecb0de5133 is 50, key is test_row_0/C:col10/1733400963201/Put/seqid=0 2024-12-05T12:16:06,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742369_1545 (size=12151) 2024-12-05T12:16:06,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401026512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:06,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401026512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:06,838 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/16662a1a077347d2a0cadcecb0de5133 2024-12-05T12:16:06,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/d4f26f134207454fa2aeaea4fe26f507 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/d4f26f134207454fa2aeaea4fe26f507 2024-12-05T12:16:06,845 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/d4f26f134207454fa2aeaea4fe26f507, entries=150, sequenceid=135, filesize=30.4 K 2024-12-05T12:16:06,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/7d8e3d714c1a4588b8b8470579044a35 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7d8e3d714c1a4588b8b8470579044a35 2024-12-05T12:16:06,848 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7d8e3d714c1a4588b8b8470579044a35, entries=150, sequenceid=135, filesize=11.9 K 2024-12-05T12:16:06,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/16662a1a077347d2a0cadcecb0de5133 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/16662a1a077347d2a0cadcecb0de5133 2024-12-05T12:16:06,852 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/16662a1a077347d2a0cadcecb0de5133, entries=150, sequenceid=135, filesize=11.9 K 2024-12-05T12:16:06,852 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 2e5f943f8689f12e595bd09f366346aa in 1674ms, sequenceid=135, compaction requested=true 2024-12-05T12:16:06,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:06,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:06,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-05T12:16:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-05T12:16:06,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-05T12:16:06,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8280 sec 2024-12-05T12:16:06,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8320 sec 2024-12-05T12:16:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T12:16:07,182 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:16:07,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:16:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-05T12:16:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T12:16:07,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:16:07,185 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:16:07,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:16:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:07,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T12:16:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:07,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055c1c0d0817834a97aa152ba85df8ab44_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400965373/Put/seqid=0 2024-12-05T12:16:07,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742370_1546 (size=14794) 2024-12-05T12:16:07,252 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:07,256 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055c1c0d0817834a97aa152ba85df8ab44_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055c1c0d0817834a97aa152ba85df8ab44_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:07,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/53697c1ac1a049589b204ee4aa4df44c, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:07,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/53697c1ac1a049589b204ee4aa4df44c is 175, key is test_row_0/A:col10/1733400965373/Put/seqid=0 2024-12-05T12:16:07,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742371_1547 (size=39749) 2024-12-05T12:16:07,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401027272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401027273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401027279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T12:16:07,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T12:16:07,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:07,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:07,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:07,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401027383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401027383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401027393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T12:16:07,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:07,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:07,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T12:16:07,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401027523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401027523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,525 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:07,525 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401027593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401027593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401027602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T12:16:07,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:07,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:07,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:07,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,661 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/53697c1ac1a049589b204ee4aa4df44c 2024-12-05T12:16:07,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/4512b20a583b415b9b82ab503e91d6c7 is 50, key is test_row_0/B:col10/1733400965373/Put/seqid=0 2024-12-05T12:16:07,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742372_1548 (size=12151) 2024-12-05T12:16:07,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T12:16:07,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:07,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:07,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:07,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T12:16:07,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401027902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401027903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401027913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:07,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T12:16:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:07,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:08,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/4512b20a583b415b9b82ab503e91d6c7 2024-12-05T12:16:08,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/eb8e1b9a00224a8d9e16f4b874a1cbcc is 50, key is test_row_0/C:col10/1733400965373/Put/seqid=0 2024-12-05T12:16:08,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742373_1549 (size=12151) 2024-12-05T12:16:08,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/eb8e1b9a00224a8d9e16f4b874a1cbcc 2024-12-05T12:16:08,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/53697c1ac1a049589b204ee4aa4df44c as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/53697c1ac1a049589b204ee4aa4df44c 2024-12-05T12:16:08,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/53697c1ac1a049589b204ee4aa4df44c, entries=200, sequenceid=147, filesize=38.8 K 2024-12-05T12:16:08,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/4512b20a583b415b9b82ab503e91d6c7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/4512b20a583b415b9b82ab503e91d6c7 2024-12-05T12:16:08,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/4512b20a583b415b9b82ab503e91d6c7, entries=150, sequenceid=147, filesize=11.9 K 2024-12-05T12:16:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/eb8e1b9a00224a8d9e16f4b874a1cbcc as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/eb8e1b9a00224a8d9e16f4b874a1cbcc 2024-12-05T12:16:08,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/eb8e1b9a00224a8d9e16f4b874a1cbcc, entries=150, sequenceid=147, filesize=11.9 K 2024-12-05T12:16:08,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2e5f943f8689f12e595bd09f366346aa in 857ms, sequenceid=147, compaction requested=true 2024-12-05T12:16:08,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:08,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:16:08,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:08,093 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-05T12:16:08,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:16:08,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:08,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:16:08,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:16:08,094 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-05T12:16:08,096 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163822 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-05T12:16:08,096 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/A is initiating minor compaction (all files) 2024-12-05T12:16:08,096 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/A in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:08,096 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/14fe5035857345e99fcdc702e4d6f2ba, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/19e8681ab25d42b594108edcfc14493c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/4a6a808219a14542910a50490790b2e1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/d4f26f134207454fa2aeaea4fe26f507, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/53697c1ac1a049589b204ee4aa4df44c] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=160.0 K 2024-12-05T12:16:08,096 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:08,096 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/14fe5035857345e99fcdc702e4d6f2ba, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/19e8681ab25d42b594108edcfc14493c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/4a6a808219a14542910a50490790b2e1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/d4f26f134207454fa2aeaea4fe26f507, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/53697c1ac1a049589b204ee4aa4df44c] 2024-12-05T12:16:08,096 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60408 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-05T12:16:08,096 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14fe5035857345e99fcdc702e4d6f2ba, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733400958901 2024-12-05T12:16:08,096 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/B is initiating minor compaction (all files) 2024-12-05T12:16:08,096 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/B in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:08,096 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7329446d75c544199261f6f1bb0e54e6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/08dddc2cf5834ffeb2c0ee7373cc38c6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9a997c9fc06e46dbb375b576d055dcc4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7d8e3d714c1a4588b8b8470579044a35, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/4512b20a583b415b9b82ab503e91d6c7] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=59.0 K 2024-12-05T12:16:08,096 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19e8681ab25d42b594108edcfc14493c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733400959039 2024-12-05T12:16:08,097 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 7329446d75c544199261f6f1bb0e54e6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733400958901 2024-12-05T12:16:08,097 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a6a808219a14542910a50490790b2e1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733400960953 2024-12-05T12:16:08,097 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4f26f134207454fa2aeaea4fe26f507, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733400963201 2024-12-05T12:16:08,097 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 08dddc2cf5834ffeb2c0ee7373cc38c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733400959039 2024-12-05T12:16:08,097 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53697c1ac1a049589b204ee4aa4df44c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733400965371 2024-12-05T12:16:08,097 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a997c9fc06e46dbb375b576d055dcc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733400960953 2024-12-05T12:16:08,098 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d8e3d714c1a4588b8b8470579044a35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733400963201 2024-12-05T12:16:08,098 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 4512b20a583b415b9b82ab503e91d6c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733400965371 2024-12-05T12:16:08,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T12:16:08,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:08,100 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T12:16:08,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:08,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:08,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:08,104 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:08,107 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205847e092fb7204ec6ba167d27c4bebff8_2e5f943f8689f12e595bd09f366346aa store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:08,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120539afa32bb4a9484f91bd8b4b177ca34b_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400967277/Put/seqid=0 2024-12-05T12:16:08,109 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#B#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:08,109 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/84877049a84c4cabb9e7d5dde3019af1 is 50, key is test_row_0/B:col10/1733400965373/Put/seqid=0 2024-12-05T12:16:08,110 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205847e092fb7204ec6ba167d27c4bebff8_2e5f943f8689f12e595bd09f366346aa, store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:08,110 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205847e092fb7204ec6ba167d27c4bebff8_2e5f943f8689f12e595bd09f366346aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:08,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742374_1550 (size=12304) 2024-12-05T12:16:08,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742376_1552 (size=4469) 2024-12-05T12:16:08,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742375_1551 (size=12425) 2024-12-05T12:16:08,145 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/84877049a84c4cabb9e7d5dde3019af1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/84877049a84c4cabb9e7d5dde3019af1 2024-12-05T12:16:08,149 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/B of 2e5f943f8689f12e595bd09f366346aa into 84877049a84c4cabb9e7d5dde3019af1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:08,149 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:08,149 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/B, priority=11, startTime=1733400968093; duration=0sec 2024-12-05T12:16:08,149 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:16:08,149 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:B 2024-12-05T12:16:08,149 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-05T12:16:08,150 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60408 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-05T12:16:08,150 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/C is initiating minor compaction (all files) 2024-12-05T12:16:08,150 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/C in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:08,151 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/695128a333e041378caaa3e5f68b4ae1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/62b6abcf4f714e3783e953e3ca96e06e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/7cb73cef679c4b8c890965784fadc108, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/16662a1a077347d2a0cadcecb0de5133, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/eb8e1b9a00224a8d9e16f4b874a1cbcc] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=59.0 K 2024-12-05T12:16:08,151 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 695128a333e041378caaa3e5f68b4ae1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733400958901 2024-12-05T12:16:08,151 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 62b6abcf4f714e3783e953e3ca96e06e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733400959039 2024-12-05T12:16:08,151 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cb73cef679c4b8c890965784fadc108, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733400960953 2024-12-05T12:16:08,151 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 16662a1a077347d2a0cadcecb0de5133, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733400963201 2024-12-05T12:16:08,152 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting eb8e1b9a00224a8d9e16f4b874a1cbcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733400965371 2024-12-05T12:16:08,159 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#C#compaction#468 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:08,159 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/e54482d202244441a259a209d98796c5 is 50, key is test_row_0/C:col10/1733400965373/Put/seqid=0 2024-12-05T12:16:08,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742377_1553 (size=12425) 2024-12-05T12:16:08,168 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/e54482d202244441a259a209d98796c5 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/e54482d202244441a259a209d98796c5 2024-12-05T12:16:08,172 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/C of 2e5f943f8689f12e595bd09f366346aa into e54482d202244441a259a209d98796c5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:08,172 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:08,172 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/C, priority=11, startTime=1733400968094; duration=0sec 2024-12-05T12:16:08,172 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:08,172 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:C 2024-12-05T12:16:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T12:16:08,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:08,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401028449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:08,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:08,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401028449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:08,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:08,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:08,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401028450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:08,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:08,523 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120539afa32bb4a9484f91bd8b4b177ca34b_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120539afa32bb4a9484f91bd8b4b177ca34b_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:08,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/7e8d6f90be514d389e13dfccc213d41a, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:08,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/7e8d6f90be514d389e13dfccc213d41a is 175, key is test_row_0/A:col10/1733400967277/Put/seqid=0 2024-12-05T12:16:08,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742378_1554 (size=31105) 2024-12-05T12:16:08,527 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/7e8d6f90be514d389e13dfccc213d41a 2024-12-05T12:16:08,532 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#A#compaction#465 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/d67b23e47e0a436d82d77aa958767db0 is 50, key is test_row_0/B:col10/1733400967277/Put/seqid=0 2024-12-05T12:16:08,533 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6390a17a07c842eca5db4d420776398f is 175, key is test_row_0/A:col10/1733400965373/Put/seqid=0 2024-12-05T12:16:08,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742379_1555 (size=12151) 2024-12-05T12:16:08,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742380_1556 (size=31379) 2024-12-05T12:16:08,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401028563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401028563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401028563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:08,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401028773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:08,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401028773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:08,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401028773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:08,936 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/d67b23e47e0a436d82d77aa958767db0 2024-12-05T12:16:08,939 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6390a17a07c842eca5db4d420776398f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6390a17a07c842eca5db4d420776398f 2024-12-05T12:16:08,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/256d424d06e34283a673dcaa64e50e76 is 50, key is test_row_0/C:col10/1733400967277/Put/seqid=0 2024-12-05T12:16:08,942 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/A of 2e5f943f8689f12e595bd09f366346aa into 6390a17a07c842eca5db4d420776398f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:08,942 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:08,942 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/A, priority=11, startTime=1733400968093; duration=0sec 2024-12-05T12:16:08,942 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:08,942 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:A 2024-12-05T12:16:08,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742381_1557 (size=12151) 2024-12-05T12:16:09,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401029082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401029082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401029082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T12:16:09,345 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/256d424d06e34283a673dcaa64e50e76 2024-12-05T12:16:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/7e8d6f90be514d389e13dfccc213d41a as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/7e8d6f90be514d389e13dfccc213d41a 2024-12-05T12:16:09,361 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/7e8d6f90be514d389e13dfccc213d41a, entries=150, sequenceid=167, filesize=30.4 K 2024-12-05T12:16:09,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/d67b23e47e0a436d82d77aa958767db0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/d67b23e47e0a436d82d77aa958767db0 2024-12-05T12:16:09,364 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/d67b23e47e0a436d82d77aa958767db0, entries=150, sequenceid=167, filesize=11.9 K 2024-12-05T12:16:09,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/256d424d06e34283a673dcaa64e50e76 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/256d424d06e34283a673dcaa64e50e76 2024-12-05T12:16:09,367 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/256d424d06e34283a673dcaa64e50e76, entries=150, sequenceid=167, filesize=11.9 K 2024-12-05T12:16:09,368 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2e5f943f8689f12e595bd09f366346aa in 1268ms, sequenceid=167, compaction requested=false 2024-12-05T12:16:09,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:09,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:09,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-05T12:16:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-05T12:16:09,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-05T12:16:09,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1840 sec 2024-12-05T12:16:09,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.1880 sec 2024-12-05T12:16:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:09,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T12:16:09,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:09,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:09,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:09,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:09,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:09,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:09,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050fab060729ce4016bf6bdd3ab1635c0c_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400969544/Put/seqid=0 2024-12-05T12:16:09,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742382_1558 (size=17284) 2024-12-05T12:16:09,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401029586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401029588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401029592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401029593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401029593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401029693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401029693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401029902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:09,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401029902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:09,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:09,958 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050fab060729ce4016bf6bdd3ab1635c0c_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050fab060729ce4016bf6bdd3ab1635c0c_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:09,958 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6ea470b4ba68474cbf3e00896a7abdf7, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:09,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6ea470b4ba68474cbf3e00896a7abdf7 is 175, key is test_row_0/A:col10/1733400969544/Put/seqid=0 2024-12-05T12:16:09,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742383_1559 (size=48389) 2024-12-05T12:16:10,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401030213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401030213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,362 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6ea470b4ba68474cbf3e00896a7abdf7 2024-12-05T12:16:10,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/e8be84607f1640a98d1e0aa58db365c1 is 50, key is test_row_0/B:col10/1733400969544/Put/seqid=0 2024-12-05T12:16:10,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742384_1560 (size=12151) 2024-12-05T12:16:10,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/e8be84607f1640a98d1e0aa58db365c1 2024-12-05T12:16:10,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/a912b4fa9c14434c8c43ad3d4e83ee57 is 50, key is test_row_0/C:col10/1733400969544/Put/seqid=0 2024-12-05T12:16:10,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742385_1561 (size=12151) 2024-12-05T12:16:10,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/a912b4fa9c14434c8c43ad3d4e83ee57 2024-12-05T12:16:10,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6ea470b4ba68474cbf3e00896a7abdf7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6ea470b4ba68474cbf3e00896a7abdf7 2024-12-05T12:16:10,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6ea470b4ba68474cbf3e00896a7abdf7, entries=250, sequenceid=184, filesize=47.3 K 2024-12-05T12:16:10,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/e8be84607f1640a98d1e0aa58db365c1 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/e8be84607f1640a98d1e0aa58db365c1 2024-12-05T12:16:10,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/e8be84607f1640a98d1e0aa58db365c1, entries=150, sequenceid=184, filesize=11.9 K 2024-12-05T12:16:10,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/a912b4fa9c14434c8c43ad3d4e83ee57 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/a912b4fa9c14434c8c43ad3d4e83ee57 2024-12-05T12:16:10,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/a912b4fa9c14434c8c43ad3d4e83ee57, entries=150, sequenceid=184, filesize=11.9 K 2024-12-05T12:16:10,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2e5f943f8689f12e595bd09f366346aa in 847ms, sequenceid=184, compaction requested=true 2024-12-05T12:16:10,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:10,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:16:10,392 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:16:10,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:10,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:16:10,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:10,392 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:16:10,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:16:10,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:16:10,394 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:16:10,394 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:16:10,394 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/B is initiating minor compaction (all files) 2024-12-05T12:16:10,394 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/A is initiating minor compaction (all files) 2024-12-05T12:16:10,394 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/A in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:10,394 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/B in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:10,394 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6390a17a07c842eca5db4d420776398f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/7e8d6f90be514d389e13dfccc213d41a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6ea470b4ba68474cbf3e00896a7abdf7] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=108.3 K 2024-12-05T12:16:10,394 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/84877049a84c4cabb9e7d5dde3019af1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/d67b23e47e0a436d82d77aa958767db0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/e8be84607f1640a98d1e0aa58db365c1] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=35.9 K 2024-12-05T12:16:10,395 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:10,395 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6390a17a07c842eca5db4d420776398f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/7e8d6f90be514d389e13dfccc213d41a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6ea470b4ba68474cbf3e00896a7abdf7] 2024-12-05T12:16:10,395 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 84877049a84c4cabb9e7d5dde3019af1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733400965371 2024-12-05T12:16:10,395 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6390a17a07c842eca5db4d420776398f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733400965371 2024-12-05T12:16:10,395 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d67b23e47e0a436d82d77aa958767db0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733400967271 2024-12-05T12:16:10,395 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e8d6f90be514d389e13dfccc213d41a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733400967271 2024-12-05T12:16:10,395 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea470b4ba68474cbf3e00896a7abdf7, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400968423 2024-12-05T12:16:10,395 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting e8be84607f1640a98d1e0aa58db365c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400968423 2024-12-05T12:16:10,400 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:10,402 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#B#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:10,402 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/bb8027e4b57543d88b1c0ba1f95f8c4d is 50, key is test_row_0/B:col10/1733400969544/Put/seqid=0 2024-12-05T12:16:10,404 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205d8bec481debf4fe18d397aec01462a05_2e5f943f8689f12e595bd09f366346aa store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:10,406 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205d8bec481debf4fe18d397aec01462a05_2e5f943f8689f12e595bd09f366346aa, store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:10,406 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d8bec481debf4fe18d397aec01462a05_2e5f943f8689f12e595bd09f366346aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:10,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742386_1562 (size=12527) 2024-12-05T12:16:10,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742387_1563 (size=4469) 2024-12-05T12:16:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:10,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T12:16:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:10,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120543720829b3ac45f5b7f3164d9825480a_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400970604/Put/seqid=0 2024-12-05T12:16:10,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742388_1564 (size=14794) 2024-12-05T12:16:10,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401030620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401030622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401030623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401030723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401030723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401030733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401030733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401030733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,825 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:10,825 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/b9dea9f0e1a1446c906fbc7f07b9aa77 is 175, key is test_row_0/A:col10/1733400969544/Put/seqid=0 2024-12-05T12:16:10,828 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/bb8027e4b57543d88b1c0ba1f95f8c4d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bb8027e4b57543d88b1c0ba1f95f8c4d 2024-12-05T12:16:10,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742389_1565 (size=31481) 2024-12-05T12:16:10,832 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/B of 2e5f943f8689f12e595bd09f366346aa into bb8027e4b57543d88b1c0ba1f95f8c4d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:10,832 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:10,832 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/B, priority=13, startTime=1733400970392; duration=0sec 2024-12-05T12:16:10,832 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:16:10,832 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:B 2024-12-05T12:16:10,832 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:16:10,833 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:16:10,833 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/C is initiating minor compaction (all files) 2024-12-05T12:16:10,833 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/C in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:10,833 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/e54482d202244441a259a209d98796c5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/256d424d06e34283a673dcaa64e50e76, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/a912b4fa9c14434c8c43ad3d4e83ee57] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=35.9 K 2024-12-05T12:16:10,833 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting e54482d202244441a259a209d98796c5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733400965371 2024-12-05T12:16:10,834 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 256d424d06e34283a673dcaa64e50e76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733400967271 2024-12-05T12:16:10,834 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting a912b4fa9c14434c8c43ad3d4e83ee57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400968423 2024-12-05T12:16:10,839 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:10,840 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/1fe2d780aa1d44a9b42c4e2c908956ee is 50, key is test_row_0/C:col10/1733400969544/Put/seqid=0 2024-12-05T12:16:10,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742390_1566 (size=12527) 2024-12-05T12:16:10,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401030942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401030942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401030943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:11,015 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:11,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120543720829b3ac45f5b7f3164d9825480a_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120543720829b3ac45f5b7f3164d9825480a_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:11,018 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6b876a004afc434eacce1a8dad32ca25, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:11,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6b876a004afc434eacce1a8dad32ca25 is 175, key is test_row_0/A:col10/1733400970604/Put/seqid=0 2024-12-05T12:16:11,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742391_1567 (size=39749) 2024-12-05T12:16:11,234 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/b9dea9f0e1a1446c906fbc7f07b9aa77 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b9dea9f0e1a1446c906fbc7f07b9aa77 2024-12-05T12:16:11,237 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/A of 2e5f943f8689f12e595bd09f366346aa into b9dea9f0e1a1446c906fbc7f07b9aa77(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:11,237 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:11,237 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/A, priority=13, startTime=1733400970392; duration=0sec 2024-12-05T12:16:11,237 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:11,237 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:A 2024-12-05T12:16:11,247 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/1fe2d780aa1d44a9b42c4e2c908956ee as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1fe2d780aa1d44a9b42c4e2c908956ee 2024-12-05T12:16:11,251 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/C of 2e5f943f8689f12e595bd09f366346aa into 1fe2d780aa1d44a9b42c4e2c908956ee(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:11,251 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:11,251 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/C, priority=13, startTime=1733400970392; duration=0sec 2024-12-05T12:16:11,251 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:11,251 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:C 2024-12-05T12:16:11,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401031253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:11,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401031253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401031253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T12:16:11,343 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:16:11,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:16:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-05T12:16:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T12:16:11,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:16:11,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:16:11,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:16:11,422 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6b876a004afc434eacce1a8dad32ca25 2024-12-05T12:16:11,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/3143a6c563a140ddb0aca3b747fb6f13 is 50, key is test_row_0/B:col10/1733400970604/Put/seqid=0 2024-12-05T12:16:11,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742392_1568 (size=12151) 2024-12-05T12:16:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T12:16:11,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T12:16:11,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:11,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:11,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:11,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:11,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:11,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T12:16:11,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:11,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:11,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:11,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:11,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T12:16:11,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401031733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:11,735 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:11,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401031743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:11,746 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:11,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401031762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401031762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:11,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401031762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:11,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T12:16:11,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:11,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:11,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:11,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:11,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:11,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/3143a6c563a140ddb0aca3b747fb6f13 2024-12-05T12:16:11,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/06b3a1e6653e4ee993a5cb24a836ae45 is 50, key is test_row_0/C:col10/1733400970604/Put/seqid=0 2024-12-05T12:16:11,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742393_1569 (size=12151) 2024-12-05T12:16:11,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/06b3a1e6653e4ee993a5cb24a836ae45 2024-12-05T12:16:11,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/6b876a004afc434eacce1a8dad32ca25 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6b876a004afc434eacce1a8dad32ca25 2024-12-05T12:16:11,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6b876a004afc434eacce1a8dad32ca25, entries=200, sequenceid=203, filesize=38.8 K 2024-12-05T12:16:11,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/3143a6c563a140ddb0aca3b747fb6f13 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/3143a6c563a140ddb0aca3b747fb6f13 2024-12-05T12:16:11,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/3143a6c563a140ddb0aca3b747fb6f13, entries=150, sequenceid=203, filesize=11.9 K 2024-12-05T12:16:11,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/06b3a1e6653e4ee993a5cb24a836ae45 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/06b3a1e6653e4ee993a5cb24a836ae45 2024-12-05T12:16:11,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/06b3a1e6653e4ee993a5cb24a836ae45, entries=150, sequenceid=203, filesize=11.9 K 2024-12-05T12:16:11,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2e5f943f8689f12e595bd09f366346aa in 1247ms, sequenceid=203, compaction requested=false 2024-12-05T12:16:11,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:11,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T12:16:11,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:11,956 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T12:16:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:11,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b727d36808794913adb11e8adac8e0fc_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400970618/Put/seqid=0 2024-12-05T12:16:11,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742394_1570 (size=12304) 2024-12-05T12:16:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T12:16:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:12,367 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b727d36808794913adb11e8adac8e0fc_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b727d36808794913adb11e8adac8e0fc_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:12,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0fcc5e88ee1045d5ad4eb726d5077e06, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:12,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0fcc5e88ee1045d5ad4eb726d5077e06 is 175, key is test_row_0/A:col10/1733400970618/Put/seqid=0 2024-12-05T12:16:12,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742395_1571 (size=31105) 2024-12-05T12:16:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T12:16:12,772 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0fcc5e88ee1045d5ad4eb726d5077e06 2024-12-05T12:16:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:12,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/5edf12a89d1f4b94b0a2a1553b4f1197 is 50, key is test_row_0/B:col10/1733400970618/Put/seqid=0 2024-12-05T12:16:12,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742396_1572 (size=12151) 2024-12-05T12:16:12,781 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/5edf12a89d1f4b94b0a2a1553b4f1197 2024-12-05T12:16:12,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/4535cb4a5eba425485cfabfdd021b2bd is 50, key is test_row_0/C:col10/1733400970618/Put/seqid=0 2024-12-05T12:16:12,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742397_1573 (size=12151) 2024-12-05T12:16:12,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401032794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:12,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401032795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:12,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401032796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401032902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401032902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401032902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401033113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401033113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401033113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,190 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/4535cb4a5eba425485cfabfdd021b2bd 2024-12-05T12:16:13,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0fcc5e88ee1045d5ad4eb726d5077e06 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0fcc5e88ee1045d5ad4eb726d5077e06 2024-12-05T12:16:13,195 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0fcc5e88ee1045d5ad4eb726d5077e06, entries=150, sequenceid=219, filesize=30.4 K 2024-12-05T12:16:13,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/5edf12a89d1f4b94b0a2a1553b4f1197 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5edf12a89d1f4b94b0a2a1553b4f1197 2024-12-05T12:16:13,198 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5edf12a89d1f4b94b0a2a1553b4f1197, entries=150, sequenceid=219, filesize=11.9 K 2024-12-05T12:16:13,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/4535cb4a5eba425485cfabfdd021b2bd as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/4535cb4a5eba425485cfabfdd021b2bd 2024-12-05T12:16:13,201 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/4535cb4a5eba425485cfabfdd021b2bd, entries=150, sequenceid=219, filesize=11.9 K 2024-12-05T12:16:13,202 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2e5f943f8689f12e595bd09f366346aa in 1246ms, sequenceid=219, compaction requested=true 2024-12-05T12:16:13,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:13,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:13,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-05T12:16:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-05T12:16:13,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-05T12:16:13,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8580 sec 2024-12-05T12:16:13,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8620 sec 2024-12-05T12:16:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:13,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T12:16:13,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:13,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:13,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:13,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:13,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:13,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:13,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120522801ab32e7049ae9e3aa2458638c947_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400973425/Put/seqid=0 2024-12-05T12:16:13,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742398_1574 (size=12304) 2024-12-05T12:16:13,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401033435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401033436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401033440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T12:16:13,503 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:16:13,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T12:16:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-05T12:16:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T12:16:13,505 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T12:16:13,505 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T12:16:13,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T12:16:13,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401033543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401033543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401033552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T12:16:13,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:13,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:13,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:13,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:13,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:13,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:13,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44166 deadline: 1733401033753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401033753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401033753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,758 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_0' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401033763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44186 deadline: 1733401033762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,767 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ea9abad0e9b,44759,1733400796278 for 'test_row_2' in 2e5f943f8689f12e595bd09f366346aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T12:16:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:13,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:13,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:13,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:13,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:13,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:13,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T12:16:13,835 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:13,838 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120522801ab32e7049ae9e3aa2458638c947_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120522801ab32e7049ae9e3aa2458638c947_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:13,838 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/a2f72d6e72754e81a283b128c01f4f53, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:13,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/a2f72d6e72754e81a283b128c01f4f53 is 175, key is test_row_0/A:col10/1733400973425/Put/seqid=0 2024-12-05T12:16:13,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742399_1575 (size=31105) 2024-12-05T12:16:13,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:13,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:13,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:13,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:13,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:13,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401034063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:14,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401034063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:14,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401034072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:14,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T12:16:14,243 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/a2f72d6e72754e81a283b128c01f4f53 2024-12-05T12:16:14,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/dffce52988434c96bc0fabbe70845d83 is 50, key is test_row_0/B:col10/1733400973425/Put/seqid=0 2024-12-05T12:16:14,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742400_1576 (size=12151) 2024-12-05T12:16:14,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,516 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T12:16:14,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401034573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:14,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401034573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:14,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401034582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T12:16:14,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/dffce52988434c96bc0fabbe70845d83 2024-12-05T12:16:14,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/d4bd93353f914b57a5148cfe7850c66f is 50, key is test_row_0/C:col10/1733400973425/Put/seqid=0 2024-12-05T12:16:14,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742401_1577 (size=12151) 2024-12-05T12:16:14,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:14,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:14,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:14,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:14,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:14,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:15,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:15,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:15,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:15,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:15,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:15,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:16:15,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/d4bd93353f914b57a5148cfe7850c66f 2024-12-05T12:16:15,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/a2f72d6e72754e81a283b128c01f4f53 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/a2f72d6e72754e81a283b128c01f4f53 2024-12-05T12:16:15,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/a2f72d6e72754e81a283b128c01f4f53, entries=150, sequenceid=240, filesize=30.4 K 2024-12-05T12:16:15,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/dffce52988434c96bc0fabbe70845d83 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/dffce52988434c96bc0fabbe70845d83 2024-12-05T12:16:15,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/dffce52988434c96bc0fabbe70845d83, entries=150, sequenceid=240, filesize=11.9 K 2024-12-05T12:16:15,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/d4bd93353f914b57a5148cfe7850c66f as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d4bd93353f914b57a5148cfe7850c66f 2024-12-05T12:16:15,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d4bd93353f914b57a5148cfe7850c66f, entries=150, sequenceid=240, filesize=11.9 K 2024-12-05T12:16:15,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 2e5f943f8689f12e595bd09f366346aa in 1648ms, sequenceid=240, compaction requested=true 2024-12-05T12:16:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:16:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:16:15,073 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:16:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:15,073 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:16:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:16:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:16:15,074 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:16:15,074 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/A is initiating minor compaction (all files) 2024-12-05T12:16:15,074 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/A in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:15,074 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b9dea9f0e1a1446c906fbc7f07b9aa77, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6b876a004afc434eacce1a8dad32ca25, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0fcc5e88ee1045d5ad4eb726d5077e06, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/a2f72d6e72754e81a283b128c01f4f53] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=130.3 K 2024-12-05T12:16:15,074 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:15,074 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b9dea9f0e1a1446c906fbc7f07b9aa77, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6b876a004afc434eacce1a8dad32ca25, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0fcc5e88ee1045d5ad4eb726d5077e06, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/a2f72d6e72754e81a283b128c01f4f53] 2024-12-05T12:16:15,074 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:16:15,075 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/B is initiating minor compaction (all files) 2024-12-05T12:16:15,075 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/B in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:15,075 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bb8027e4b57543d88b1c0ba1f95f8c4d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/3143a6c563a140ddb0aca3b747fb6f13, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5edf12a89d1f4b94b0a2a1553b4f1197, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/dffce52988434c96bc0fabbe70845d83] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=47.8 K 2024-12-05T12:16:15,079 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9dea9f0e1a1446c906fbc7f07b9aa77, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400968423 2024-12-05T12:16:15,079 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting bb8027e4b57543d88b1c0ba1f95f8c4d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400968423 2024-12-05T12:16:15,079 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b876a004afc434eacce1a8dad32ca25, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733400969577 2024-12-05T12:16:15,079 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 3143a6c563a140ddb0aca3b747fb6f13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733400969582 2024-12-05T12:16:15,079 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fcc5e88ee1045d5ad4eb726d5077e06, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733400970618 2024-12-05T12:16:15,079 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 5edf12a89d1f4b94b0a2a1553b4f1197, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733400970618 2024-12-05T12:16:15,080 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting dffce52988434c96bc0fabbe70845d83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733400972792 2024-12-05T12:16:15,080 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f72d6e72754e81a283b128c01f4f53, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733400972792 2024-12-05T12:16:15,087 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#B#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:15,087 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/5b71a246a33745ce829d31290fe058af is 50, key is test_row_0/B:col10/1733400973425/Put/seqid=0 2024-12-05T12:16:15,087 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:15,089 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205f2f0ca011f244904bf76fececbc8c75b_2e5f943f8689f12e595bd09f366346aa store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:15,091 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205f2f0ca011f244904bf76fececbc8c75b_2e5f943f8689f12e595bd09f366346aa, store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:15,091 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f2f0ca011f244904bf76fececbc8c75b_2e5f943f8689f12e595bd09f366346aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:15,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742402_1578 (size=12663) 2024-12-05T12:16:15,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742403_1579 (size=4469) 2024-12-05T12:16:15,095 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#A#compaction#487 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:15,095 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/ac54095178ff4384af779ff3e754a249 is 175, key is test_row_0/A:col10/1733400973425/Put/seqid=0 2024-12-05T12:16:15,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742404_1580 (size=31617) 2024-12-05T12:16:15,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44759 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T12:16:15,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:15,184 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T12:16:15,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:15,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:15,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:15,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:15,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:15,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:15,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120509f45aef917a49eeba1eab8ac056f5aa_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400973439/Put/seqid=0 2024-12-05T12:16:15,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742405_1581 (size=12304) 2024-12-05T12:16:15,497 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/5b71a246a33745ce829d31290fe058af as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5b71a246a33745ce829d31290fe058af 2024-12-05T12:16:15,500 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/B of 2e5f943f8689f12e595bd09f366346aa into 5b71a246a33745ce829d31290fe058af(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:15,500 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:15,500 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/B, priority=12, startTime=1733400975073; duration=0sec 2024-12-05T12:16:15,500 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:16:15,501 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:B 2024-12-05T12:16:15,501 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T12:16:15,501 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/ac54095178ff4384af779ff3e754a249 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/ac54095178ff4384af779ff3e754a249 2024-12-05T12:16:15,501 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T12:16:15,501 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/C is initiating minor compaction (all files) 2024-12-05T12:16:15,501 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/C in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:15,501 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1fe2d780aa1d44a9b42c4e2c908956ee, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/06b3a1e6653e4ee993a5cb24a836ae45, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/4535cb4a5eba425485cfabfdd021b2bd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d4bd93353f914b57a5148cfe7850c66f] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=47.8 K 2024-12-05T12:16:15,502 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe2d780aa1d44a9b42c4e2c908956ee, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733400968423 2024-12-05T12:16:15,502 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b3a1e6653e4ee993a5cb24a836ae45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733400969582 2024-12-05T12:16:15,502 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 4535cb4a5eba425485cfabfdd021b2bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733400970618 2024-12-05T12:16:15,503 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d4bd93353f914b57a5148cfe7850c66f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733400972792 2024-12-05T12:16:15,504 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/A of 2e5f943f8689f12e595bd09f366346aa into ac54095178ff4384af779ff3e754a249(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:15,504 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:15,504 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/A, priority=12, startTime=1733400975073; duration=0sec 2024-12-05T12:16:15,504 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:15,504 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:A 2024-12-05T12:16:15,509 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#C#compaction#489 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:15,509 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/d5945734c237460faf0210af76f4775d is 50, key is test_row_0/C:col10/1733400973425/Put/seqid=0 2024-12-05T12:16:15,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742406_1582 (size=12663) 2024-12-05T12:16:15,516 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/d5945734c237460faf0210af76f4775d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d5945734c237460faf0210af76f4775d 2024-12-05T12:16:15,519 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/C of 2e5f943f8689f12e595bd09f366346aa into d5945734c237460faf0210af76f4775d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:15,519 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:15,519 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/C, priority=12, startTime=1733400975073; duration=0sec 2024-12-05T12:16:15,519 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:15,519 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:C 2024-12-05T12:16:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. as already flushing 2024-12-05T12:16:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:15,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:15,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120509f45aef917a49eeba1eab8ac056f5aa_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120509f45aef917a49eeba1eab8ac056f5aa_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:15,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/8f30215e8a614f8ca1739b4256b65146, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:15,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/8f30215e8a614f8ca1739b4256b65146 is 175, key is test_row_0/A:col10/1733400973439/Put/seqid=0 2024-12-05T12:16:15,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742407_1583 (size=31105) 2024-12-05T12:16:15,599 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/8f30215e8a614f8ca1739b4256b65146 2024-12-05T12:16:15,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/657b0dbb86cd4cdcac7f412976ae06a3 is 50, key is test_row_0/B:col10/1733400973439/Put/seqid=0 2024-12-05T12:16:15,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401035607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:15,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:15,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400975599, max timestamp=1733400975608Num uniques 859; 2024-12-05T12:16:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401035608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:15,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400975599, max timestamp=1733400975608Num uniques 859; 2024-12-05T12:16:15,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:16:15,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:16:15,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733400975599, max timestamp=1733400975608Num uniques 859; 2024-12-05T12:16:15,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ea9abad0e9b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T12:16:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:15,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742408_1584 (size=12151) 2024-12-05T12:16:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T12:16:15,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401035655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:15,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:15,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401035713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401035713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:15,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401035763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:15,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401035923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:15,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401035923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:15,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401035972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:16,013 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/657b0dbb86cd4cdcac7f412976ae06a3 2024-12-05T12:16:16,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/1f94d4d941514f559c162b608565eaaf is 50, key is test_row_0/C:col10/1733400973439/Put/seqid=0 2024-12-05T12:16:16,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742409_1585 (size=12151) 2024-12-05T12:16:16,023 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/1f94d4d941514f559c162b608565eaaf 2024-12-05T12:16:16,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/8f30215e8a614f8ca1739b4256b65146 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/8f30215e8a614f8ca1739b4256b65146 2024-12-05T12:16:16,029 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/8f30215e8a614f8ca1739b4256b65146, entries=150, sequenceid=251, filesize=30.4 K 2024-12-05T12:16:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/657b0dbb86cd4cdcac7f412976ae06a3 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/657b0dbb86cd4cdcac7f412976ae06a3 2024-12-05T12:16:16,032 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/657b0dbb86cd4cdcac7f412976ae06a3, entries=150, sequenceid=251, filesize=11.9 K 2024-12-05T12:16:16,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/1f94d4d941514f559c162b608565eaaf as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1f94d4d941514f559c162b608565eaaf 2024-12-05T12:16:16,036 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1f94d4d941514f559c162b608565eaaf, entries=150, sequenceid=251, filesize=11.9 K 2024-12-05T12:16:16,036 INFO [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2e5f943f8689f12e595bd09f366346aa in 852ms, sequenceid=251, compaction requested=false 2024-12-05T12:16:16,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:16,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:16,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ea9abad0e9b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-05T12:16:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-05T12:16:16,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-05T12:16:16,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5330 sec 2024-12-05T12:16:16,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.5360 sec 2024-12-05T12:16:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:16,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-05T12:16:16,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:16,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:16,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:16,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:16,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:16,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T12:16:16,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050984c953bd924a4484e9e4ac8fe3364b_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400976234/Put/seqid=0 2024-12-05T12:16:16,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742410_1586 (size=12454) 2024-12-05T12:16:16,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401036247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:16,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401036249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:16,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401036283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:16,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401036363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:16,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401036363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:16,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401036573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:16,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401036573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:16,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:16,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:16,648 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:16,651 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050984c953bd924a4484e9e4ac8fe3364b_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050984c953bd924a4484e9e4ac8fe3364b_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:16,652 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0f1cf408209e4980ae84d36f52681a73, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:16,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0f1cf408209e4980ae84d36f52681a73 is 175, key is test_row_0/A:col10/1733400976234/Put/seqid=0 2024-12-05T12:16:16,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742411_1587 (size=31255) 2024-12-05T12:16:16,655 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0f1cf408209e4980ae84d36f52681a73 2024-12-05T12:16:16,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/bd4e7f40b6244f41940363c8e6a8a8c7 is 50, key is test_row_0/B:col10/1733400976234/Put/seqid=0 2024-12-05T12:16:16,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742412_1588 (size=12301) 2024-12-05T12:16:16,699 INFO [Thread-2327 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2327. 2024-12-05T12:16:16,699 DEBUG [Thread-2327 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:16:16,699 DEBUG [Thread-2327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:16,699 DEBUG [Thread-2327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:16,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:16,700 INFO [Thread-2329 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2329. 2024-12-05T12:16:16,700 DEBUG [Thread-2329 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:16:16,700 INFO [Thread-2333 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2333. 2024-12-05T12:16:16,700 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:16,700 DEBUG [Thread-2333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:16:16,700 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:16,700 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:16,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:16,700 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:16,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:16,702 INFO [Thread-2335 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2335. 2024-12-05T12:16:16,702 DEBUG [Thread-2335 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:16:16,702 DEBUG [Thread-2335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:16,702 DEBUG [Thread-2335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:16,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:16,703 INFO [Thread-2331 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2331. 2024-12-05T12:16:16,703 DEBUG [Thread-2331 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:16:16,703 DEBUG [Thread-2331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:16,703 DEBUG [Thread-2331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:16,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:16,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44212 deadline: 1733401036793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:16,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:16,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:16,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401036884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401036884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:16,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:16,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:16,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:17,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/bd4e7f40b6244f41940363c8e6a8a8c7 2024-12-05T12:16:17,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/8b2b7d36383541cd861a0d3f40a2e0b0 is 50, key is test_row_0/C:col10/1733400976234/Put/seqid=0 2024-12-05T12:16:17,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742413_1589 (size=12301) 2024-12-05T12:16:17,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:17,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T12:16:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44234 deadline: 1733401037394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44759 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44196 deadline: 1733401037394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 , the old value is region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T12:16:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., hostname=9ea9abad0e9b,44759,1733400796278, seqNum=5 because the exception is null or not the one we care about 2024-12-05T12:16:17,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/8b2b7d36383541cd861a0d3f40a2e0b0 2024-12-05T12:16:17,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0f1cf408209e4980ae84d36f52681a73 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0f1cf408209e4980ae84d36f52681a73 2024-12-05T12:16:17,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0f1cf408209e4980ae84d36f52681a73, entries=150, sequenceid=289, filesize=30.5 K 2024-12-05T12:16:17,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/bd4e7f40b6244f41940363c8e6a8a8c7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bd4e7f40b6244f41940363c8e6a8a8c7 2024-12-05T12:16:17,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bd4e7f40b6244f41940363c8e6a8a8c7, entries=150, sequenceid=289, filesize=12.0 K 2024-12-05T12:16:17,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/8b2b7d36383541cd861a0d3f40a2e0b0 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/8b2b7d36383541cd861a0d3f40a2e0b0 2024-12-05T12:16:17,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/8b2b7d36383541cd861a0d3f40a2e0b0, entries=150, sequenceid=289, filesize=12.0 K 2024-12-05T12:16:17,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 2e5f943f8689f12e595bd09f366346aa in 1261ms, sequenceid=289, compaction requested=true 2024-12-05T12:16:17,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:17,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T12:16:17,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:17,497 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:16:17,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T12:16:17,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:17,497 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:16:17,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e5f943f8689f12e595bd09f366346aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T12:16:17,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:16:17,497 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:16:17,497 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93977 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:16:17,497 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/A is initiating minor compaction (all files) 2024-12-05T12:16:17,497 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/B is initiating minor compaction (all files) 2024-12-05T12:16:17,497 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/A in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:17,497 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/B in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:17,497 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5b71a246a33745ce829d31290fe058af, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/657b0dbb86cd4cdcac7f412976ae06a3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bd4e7f40b6244f41940363c8e6a8a8c7] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=36.2 K 2024-12-05T12:16:17,497 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/ac54095178ff4384af779ff3e754a249, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/8f30215e8a614f8ca1739b4256b65146, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0f1cf408209e4980ae84d36f52681a73] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=91.8 K 2024-12-05T12:16:17,497 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:17,497 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. files: [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/ac54095178ff4384af779ff3e754a249, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/8f30215e8a614f8ca1739b4256b65146, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0f1cf408209e4980ae84d36f52681a73] 2024-12-05T12:16:17,498 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b71a246a33745ce829d31290fe058af, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733400972792 2024-12-05T12:16:17,498 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac54095178ff4384af779ff3e754a249, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733400972792 2024-12-05T12:16:17,498 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 657b0dbb86cd4cdcac7f412976ae06a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733400973433 2024-12-05T12:16:17,498 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f30215e8a614f8ca1739b4256b65146, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733400973433 2024-12-05T12:16:17,498 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting bd4e7f40b6244f41940363c8e6a8a8c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733400975646 2024-12-05T12:16:17,498 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1cf408209e4980ae84d36f52681a73, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733400975646 2024-12-05T12:16:17,502 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:17,503 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:17,503 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/91d748aa37234ccea7673e6538144b1d is 50, key is test_row_0/B:col10/1733400976234/Put/seqid=0 2024-12-05T12:16:17,503 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412059fb20db6ae1e4fe9a8b90ff8b68a03b6_2e5f943f8689f12e595bd09f366346aa store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:17,505 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412059fb20db6ae1e4fe9a8b90ff8b68a03b6_2e5f943f8689f12e595bd09f366346aa, store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:17,506 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059fb20db6ae1e4fe9a8b90ff8b68a03b6_2e5f943f8689f12e595bd09f366346aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:17,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742414_1590 (size=12915) 2024-12-05T12:16:17,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742415_1591 (size=4469) 2024-12-05T12:16:17,509 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/91d748aa37234ccea7673e6538144b1d as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/91d748aa37234ccea7673e6538144b1d 2024-12-05T12:16:17,512 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/B of 2e5f943f8689f12e595bd09f366346aa into 91d748aa37234ccea7673e6538144b1d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:17,512 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:17,512 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/B, priority=13, startTime=1733400977497; duration=0sec 2024-12-05T12:16:17,512 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T12:16:17,512 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:B 2024-12-05T12:16:17,512 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T12:16:17,513 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T12:16:17,513 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1541): 2e5f943f8689f12e595bd09f366346aa/C is initiating minor compaction (all files) 2024-12-05T12:16:17,513 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e5f943f8689f12e595bd09f366346aa/C in TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:17,513 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d5945734c237460faf0210af76f4775d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1f94d4d941514f559c162b608565eaaf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/8b2b7d36383541cd861a0d3f40a2e0b0] into tmpdir=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp, totalSize=36.2 K 2024-12-05T12:16:17,513 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting d5945734c237460faf0210af76f4775d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733400972792 2024-12-05T12:16:17,513 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f94d4d941514f559c162b608565eaaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733400973433 2024-12-05T12:16:17,514 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b2b7d36383541cd861a0d3f40a2e0b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733400975646 2024-12-05T12:16:17,518 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#C#compaction#497 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:17,518 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/01ee0300e74941e1a2684b1bbd77e8e7 is 50, key is test_row_0/C:col10/1733400976234/Put/seqid=0 2024-12-05T12:16:17,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742416_1592 (size=12915) 2024-12-05T12:16:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T12:16:17,675 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:16:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44759 {}] regionserver.HRegion(8855): Flush requested on 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:17,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T12:16:17,799 INFO [Thread-2320 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2320. 2024-12-05T12:16:17,799 INFO [Thread-2316 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2316. 2024-12-05T12:16:17,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:17,799 DEBUG [Thread-2320 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:16:17,799 DEBUG [Thread-2316 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:16:17,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:17,799 DEBUG [Thread-2320 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:17,799 DEBUG [Thread-2316 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:17,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:17,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:17,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:17,799 DEBUG [Thread-2316 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:17,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:17,799 DEBUG [Thread-2320 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:17,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:17,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:17,805 INFO [Thread-2322 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2322. 2024-12-05T12:16:17,805 DEBUG [Thread-2322 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:16:17,805 DEBUG [Thread-2322 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:17,806 DEBUG [Thread-2322 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:17,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:17,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059b7903e3c43740a8a7fd18ecceac2b40_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400977794/Put/seqid=0 2024-12-05T12:16:17,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742417_1593 (size=12454) 2024-12-05T12:16:17,911 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e5f943f8689f12e595bd09f366346aa#A#compaction#496 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T12:16:17,913 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/3468f4a311404704b9bcd7e434512772 is 175, key is test_row_0/A:col10/1733400976234/Put/seqid=0 2024-12-05T12:16:17,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742418_1594 (size=31869) 2024-12-05T12:16:17,925 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/01ee0300e74941e1a2684b1bbd77e8e7 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/01ee0300e74941e1a2684b1bbd77e8e7 2024-12-05T12:16:17,928 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/C of 2e5f943f8689f12e595bd09f366346aa into 01ee0300e74941e1a2684b1bbd77e8e7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:17,928 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:17,928 INFO [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/C, priority=13, startTime=1733400977497; duration=0sec 2024-12-05T12:16:17,928 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:17,928 DEBUG [RS:0;9ea9abad0e9b:44759-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:C 2024-12-05T12:16:18,213 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:18,220 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059b7903e3c43740a8a7fd18ecceac2b40_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059b7903e3c43740a8a7fd18ecceac2b40_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:18,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0d0c5c68aded4e1280f373fdc0ea4167, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:18,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0d0c5c68aded4e1280f373fdc0ea4167 is 175, key is test_row_0/A:col10/1733400977794/Put/seqid=0 2024-12-05T12:16:18,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742419_1595 (size=31255) 2024-12-05T12:16:18,329 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/3468f4a311404704b9bcd7e434512772 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/3468f4a311404704b9bcd7e434512772 2024-12-05T12:16:18,333 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e5f943f8689f12e595bd09f366346aa/A of 2e5f943f8689f12e595bd09f366346aa into 3468f4a311404704b9bcd7e434512772(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T12:16:18,333 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:18,333 INFO [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., storeName=2e5f943f8689f12e595bd09f366346aa/A, priority=13, startTime=1733400977496; duration=0sec 2024-12-05T12:16:18,333 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T12:16:18,333 DEBUG [RS:0;9ea9abad0e9b:44759-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e5f943f8689f12e595bd09f366346aa:A 2024-12-05T12:16:18,408 INFO [Thread-2324 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2324. 2024-12-05T12:16:18,408 DEBUG [Thread-2324 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:16:18,408 DEBUG [Thread-2324 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:18,408 DEBUG [Thread-2324 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:18,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:18,414 INFO [Thread-2318 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2318. 2024-12-05T12:16:18,414 DEBUG [Thread-2318 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T12:16:18,414 DEBUG [Thread-2318 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:18,414 DEBUG [Thread-2318 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:18,414 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4405 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4362 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4491 2024-12-05T12:16:18,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4391 2024-12-05T12:16:18,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4335 2024-12-05T12:16:18,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T12:16:18,415 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:16:18,415 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:16:18,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:18,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:18,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:18,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T12:16:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T12:16:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T12:16:18,418 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400978418"}]},"ts":"1733400978418"} 2024-12-05T12:16:18,419 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T12:16:18,420 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T12:16:18,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T12:16:18,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, UNASSIGN}] 2024-12-05T12:16:18,422 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, UNASSIGN 2024-12-05T12:16:18,423 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=2e5f943f8689f12e595bd09f366346aa, regionState=CLOSING, regionLocation=9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:18,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, UNASSIGN because future has completed 2024-12-05T12:16:18,424 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T12:16:18,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278}] 2024-12-05T12:16:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T12:16:18,577 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:18,577 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T12:16:18,577 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 2e5f943f8689f12e595bd09f366346aa, disabling compactions & flushes 2024-12-05T12:16:18,578 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:18,629 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0d0c5c68aded4e1280f373fdc0ea4167 2024-12-05T12:16:18,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/9d7523561ff8436a9e9fbee2bfee3b5b is 50, key is test_row_0/B:col10/1733400977794/Put/seqid=0 2024-12-05T12:16:18,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742420_1596 (size=12301) 2024-12-05T12:16:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T12:16:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T12:16:19,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/9d7523561ff8436a9e9fbee2bfee3b5b 2024-12-05T12:16:19,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/3a4c2304c76d4e5a9668e49a9be77f11 is 50, key is test_row_0/C:col10/1733400977794/Put/seqid=0 2024-12-05T12:16:19,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742421_1597 (size=12301) 2024-12-05T12:16:19,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/3a4c2304c76d4e5a9668e49a9be77f11 2024-12-05T12:16:19,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/0d0c5c68aded4e1280f373fdc0ea4167 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0d0c5c68aded4e1280f373fdc0ea4167 2024-12-05T12:16:19,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0d0c5c68aded4e1280f373fdc0ea4167, entries=150, sequenceid=302, filesize=30.5 K 2024-12-05T12:16:19,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/9d7523561ff8436a9e9fbee2bfee3b5b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9d7523561ff8436a9e9fbee2bfee3b5b 2024-12-05T12:16:19,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9d7523561ff8436a9e9fbee2bfee3b5b, entries=150, sequenceid=302, filesize=12.0 K 2024-12-05T12:16:19,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/3a4c2304c76d4e5a9668e49a9be77f11 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/3a4c2304c76d4e5a9668e49a9be77f11 2024-12-05T12:16:19,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/3a4c2304c76d4e5a9668e49a9be77f11, entries=150, sequenceid=302, filesize=12.0 K 2024-12-05T12:16:19,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 2e5f943f8689f12e595bd09f366346aa in 1698ms, sequenceid=302, compaction requested=false 2024-12-05T12:16:19,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e5f943f8689f12e595bd09f366346aa: 2024-12-05T12:16:19,496 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:19,496 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:19,496 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. after waiting 0 ms 2024-12-05T12:16:19,496 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:19,496 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 2e5f943f8689f12e595bd09f366346aa 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-05T12:16:19,496 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=A 2024-12-05T12:16:19,496 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:19,496 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=B 2024-12-05T12:16:19,496 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:19,496 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e5f943f8689f12e595bd09f366346aa, store=C 2024-12-05T12:16:19,496 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T12:16:19,500 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cb26e1ae28d3485f843fc23e40e04611_2e5f943f8689f12e595bd09f366346aa is 50, key is test_row_0/A:col10/1733400978404/Put/seqid=0 2024-12-05T12:16:19,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742422_1598 (size=9914) 2024-12-05T12:16:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T12:16:19,905 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:16:19,914 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cb26e1ae28d3485f843fc23e40e04611_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cb26e1ae28d3485f843fc23e40e04611_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:19,915 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/82632152efdc4435b35f34487eae318b, store: [table=TestAcidGuarantees family=A region=2e5f943f8689f12e595bd09f366346aa] 2024-12-05T12:16:19,915 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/82632152efdc4435b35f34487eae318b is 175, key is test_row_0/A:col10/1733400978404/Put/seqid=0 2024-12-05T12:16:19,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742423_1599 (size=22561) 2024-12-05T12:16:20,321 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/82632152efdc4435b35f34487eae318b 2024-12-05T12:16:20,334 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/9d30493753794189a6ecb7af8fc7b58e is 50, key is test_row_0/B:col10/1733400978404/Put/seqid=0 2024-12-05T12:16:20,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742424_1600 (size=9857) 2024-12-05T12:16:20,339 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/9d30493753794189a6ecb7af8fc7b58e 2024-12-05T12:16:20,345 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/b59f53783bf7462b8a6f2566e020f1cf is 50, key is test_row_0/C:col10/1733400978404/Put/seqid=0 2024-12-05T12:16:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742425_1601 (size=9857) 2024-12-05T12:16:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T12:16:20,750 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/b59f53783bf7462b8a6f2566e020f1cf 2024-12-05T12:16:20,757 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/A/82632152efdc4435b35f34487eae318b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/82632152efdc4435b35f34487eae318b 2024-12-05T12:16:20,761 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/82632152efdc4435b35f34487eae318b, entries=100, sequenceid=310, filesize=22.0 K 2024-12-05T12:16:20,762 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/B/9d30493753794189a6ecb7af8fc7b58e as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9d30493753794189a6ecb7af8fc7b58e 2024-12-05T12:16:20,766 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9d30493753794189a6ecb7af8fc7b58e, entries=100, sequenceid=310, filesize=9.6 K 2024-12-05T12:16:20,767 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/.tmp/C/b59f53783bf7462b8a6f2566e020f1cf as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/b59f53783bf7462b8a6f2566e020f1cf 2024-12-05T12:16:20,769 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/b59f53783bf7462b8a6f2566e020f1cf, entries=100, sequenceid=310, filesize=9.6 K 2024-12-05T12:16:20,770 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 2e5f943f8689f12e595bd09f366346aa in 1274ms, sequenceid=310, compaction requested=true 2024-12-05T12:16:20,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/afc25a716e664a9cbca24777b2bc38cf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b14b95406530422e8da6244ba1cea20e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6bc80f301f804bee8b96d7b69286d79d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/14fe5035857345e99fcdc702e4d6f2ba, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/19e8681ab25d42b594108edcfc14493c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/4a6a808219a14542910a50490790b2e1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/d4f26f134207454fa2aeaea4fe26f507, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/53697c1ac1a049589b204ee4aa4df44c, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6390a17a07c842eca5db4d420776398f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/7e8d6f90be514d389e13dfccc213d41a, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6ea470b4ba68474cbf3e00896a7abdf7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b9dea9f0e1a1446c906fbc7f07b9aa77, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6b876a004afc434eacce1a8dad32ca25, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0fcc5e88ee1045d5ad4eb726d5077e06, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/ac54095178ff4384af779ff3e754a249, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/a2f72d6e72754e81a283b128c01f4f53, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/8f30215e8a614f8ca1739b4256b65146, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0f1cf408209e4980ae84d36f52681a73] to archive 2024-12-05T12:16:20,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:16:20,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/afc25a716e664a9cbca24777b2bc38cf to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/afc25a716e664a9cbca24777b2bc38cf 2024-12-05T12:16:20,773 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b14b95406530422e8da6244ba1cea20e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b14b95406530422e8da6244ba1cea20e 2024-12-05T12:16:20,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6bc80f301f804bee8b96d7b69286d79d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6bc80f301f804bee8b96d7b69286d79d 2024-12-05T12:16:20,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/14fe5035857345e99fcdc702e4d6f2ba to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/14fe5035857345e99fcdc702e4d6f2ba 2024-12-05T12:16:20,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/19e8681ab25d42b594108edcfc14493c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/19e8681ab25d42b594108edcfc14493c 2024-12-05T12:16:20,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/4a6a808219a14542910a50490790b2e1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/4a6a808219a14542910a50490790b2e1 2024-12-05T12:16:20,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/d4f26f134207454fa2aeaea4fe26f507 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/d4f26f134207454fa2aeaea4fe26f507 2024-12-05T12:16:20,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/53697c1ac1a049589b204ee4aa4df44c to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/53697c1ac1a049589b204ee4aa4df44c 2024-12-05T12:16:20,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6390a17a07c842eca5db4d420776398f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6390a17a07c842eca5db4d420776398f 2024-12-05T12:16:20,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/7e8d6f90be514d389e13dfccc213d41a to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/7e8d6f90be514d389e13dfccc213d41a 2024-12-05T12:16:20,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6ea470b4ba68474cbf3e00896a7abdf7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6ea470b4ba68474cbf3e00896a7abdf7 2024-12-05T12:16:20,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b9dea9f0e1a1446c906fbc7f07b9aa77 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/b9dea9f0e1a1446c906fbc7f07b9aa77 2024-12-05T12:16:20,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6b876a004afc434eacce1a8dad32ca25 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/6b876a004afc434eacce1a8dad32ca25 2024-12-05T12:16:20,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0fcc5e88ee1045d5ad4eb726d5077e06 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0fcc5e88ee1045d5ad4eb726d5077e06 2024-12-05T12:16:20,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/ac54095178ff4384af779ff3e754a249 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/ac54095178ff4384af779ff3e754a249 2024-12-05T12:16:20,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/a2f72d6e72754e81a283b128c01f4f53 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/a2f72d6e72754e81a283b128c01f4f53 2024-12-05T12:16:20,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/8f30215e8a614f8ca1739b4256b65146 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/8f30215e8a614f8ca1739b4256b65146 2024-12-05T12:16:20,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0f1cf408209e4980ae84d36f52681a73 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0f1cf408209e4980ae84d36f52681a73 2024-12-05T12:16:20,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/916e6fc4872d44ca969a760dc7c86c46, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/0b45c1a4d8454610a6c94ced1665aed1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7329446d75c544199261f6f1bb0e54e6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/b4c5ac81e6f44c8c9616c10fe3bb8a78, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/08dddc2cf5834ffeb2c0ee7373cc38c6, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9a997c9fc06e46dbb375b576d055dcc4, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7d8e3d714c1a4588b8b8470579044a35, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/84877049a84c4cabb9e7d5dde3019af1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/4512b20a583b415b9b82ab503e91d6c7, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/d67b23e47e0a436d82d77aa958767db0, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bb8027e4b57543d88b1c0ba1f95f8c4d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/e8be84607f1640a98d1e0aa58db365c1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/3143a6c563a140ddb0aca3b747fb6f13, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5edf12a89d1f4b94b0a2a1553b4f1197, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5b71a246a33745ce829d31290fe058af, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/dffce52988434c96bc0fabbe70845d83, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/657b0dbb86cd4cdcac7f412976ae06a3, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bd4e7f40b6244f41940363c8e6a8a8c7] to archive 2024-12-05T12:16:20,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:16:20,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/916e6fc4872d44ca969a760dc7c86c46 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/916e6fc4872d44ca969a760dc7c86c46 2024-12-05T12:16:20,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/0b45c1a4d8454610a6c94ced1665aed1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/0b45c1a4d8454610a6c94ced1665aed1 2024-12-05T12:16:20,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7329446d75c544199261f6f1bb0e54e6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7329446d75c544199261f6f1bb0e54e6 2024-12-05T12:16:20,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/b4c5ac81e6f44c8c9616c10fe3bb8a78 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/b4c5ac81e6f44c8c9616c10fe3bb8a78 2024-12-05T12:16:20,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/08dddc2cf5834ffeb2c0ee7373cc38c6 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/08dddc2cf5834ffeb2c0ee7373cc38c6 2024-12-05T12:16:20,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9a997c9fc06e46dbb375b576d055dcc4 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9a997c9fc06e46dbb375b576d055dcc4 2024-12-05T12:16:20,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7d8e3d714c1a4588b8b8470579044a35 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/7d8e3d714c1a4588b8b8470579044a35 2024-12-05T12:16:20,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/84877049a84c4cabb9e7d5dde3019af1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/84877049a84c4cabb9e7d5dde3019af1 2024-12-05T12:16:20,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/4512b20a583b415b9b82ab503e91d6c7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/4512b20a583b415b9b82ab503e91d6c7 2024-12-05T12:16:20,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/d67b23e47e0a436d82d77aa958767db0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/d67b23e47e0a436d82d77aa958767db0 2024-12-05T12:16:20,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bb8027e4b57543d88b1c0ba1f95f8c4d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bb8027e4b57543d88b1c0ba1f95f8c4d 2024-12-05T12:16:20,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/e8be84607f1640a98d1e0aa58db365c1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/e8be84607f1640a98d1e0aa58db365c1 2024-12-05T12:16:20,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/3143a6c563a140ddb0aca3b747fb6f13 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/3143a6c563a140ddb0aca3b747fb6f13 2024-12-05T12:16:20,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5edf12a89d1f4b94b0a2a1553b4f1197 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5edf12a89d1f4b94b0a2a1553b4f1197 2024-12-05T12:16:20,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5b71a246a33745ce829d31290fe058af to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/5b71a246a33745ce829d31290fe058af 2024-12-05T12:16:20,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/dffce52988434c96bc0fabbe70845d83 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/dffce52988434c96bc0fabbe70845d83 2024-12-05T12:16:20,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/657b0dbb86cd4cdcac7f412976ae06a3 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/657b0dbb86cd4cdcac7f412976ae06a3 2024-12-05T12:16:20,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bd4e7f40b6244f41940363c8e6a8a8c7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/bd4e7f40b6244f41940363c8e6a8a8c7 2024-12-05T12:16:20,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1a0248970862480fb0dc7a01c6f778b8, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/aab05b6657794599be46411ec01cf248, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/695128a333e041378caaa3e5f68b4ae1, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/88d528baaab64265a2effef0546f7381, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/62b6abcf4f714e3783e953e3ca96e06e, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/7cb73cef679c4b8c890965784fadc108, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/16662a1a077347d2a0cadcecb0de5133, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/e54482d202244441a259a209d98796c5, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/eb8e1b9a00224a8d9e16f4b874a1cbcc, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/256d424d06e34283a673dcaa64e50e76, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1fe2d780aa1d44a9b42c4e2c908956ee, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/a912b4fa9c14434c8c43ad3d4e83ee57, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/06b3a1e6653e4ee993a5cb24a836ae45, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/4535cb4a5eba425485cfabfdd021b2bd, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d5945734c237460faf0210af76f4775d, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d4bd93353f914b57a5148cfe7850c66f, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1f94d4d941514f559c162b608565eaaf, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/8b2b7d36383541cd861a0d3f40a2e0b0] to archive 2024-12-05T12:16:20,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T12:16:20,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1a0248970862480fb0dc7a01c6f778b8 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1a0248970862480fb0dc7a01c6f778b8 2024-12-05T12:16:20,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/aab05b6657794599be46411ec01cf248 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/aab05b6657794599be46411ec01cf248 2024-12-05T12:16:20,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/695128a333e041378caaa3e5f68b4ae1 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/695128a333e041378caaa3e5f68b4ae1 2024-12-05T12:16:20,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/88d528baaab64265a2effef0546f7381 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/88d528baaab64265a2effef0546f7381 2024-12-05T12:16:20,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/62b6abcf4f714e3783e953e3ca96e06e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/62b6abcf4f714e3783e953e3ca96e06e 2024-12-05T12:16:20,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/7cb73cef679c4b8c890965784fadc108 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/7cb73cef679c4b8c890965784fadc108 2024-12-05T12:16:20,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/16662a1a077347d2a0cadcecb0de5133 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/16662a1a077347d2a0cadcecb0de5133 2024-12-05T12:16:20,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/e54482d202244441a259a209d98796c5 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/e54482d202244441a259a209d98796c5 2024-12-05T12:16:20,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/eb8e1b9a00224a8d9e16f4b874a1cbcc to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/eb8e1b9a00224a8d9e16f4b874a1cbcc 2024-12-05T12:16:20,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/256d424d06e34283a673dcaa64e50e76 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/256d424d06e34283a673dcaa64e50e76 2024-12-05T12:16:20,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1fe2d780aa1d44a9b42c4e2c908956ee to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1fe2d780aa1d44a9b42c4e2c908956ee 2024-12-05T12:16:20,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/a912b4fa9c14434c8c43ad3d4e83ee57 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/a912b4fa9c14434c8c43ad3d4e83ee57 2024-12-05T12:16:20,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/06b3a1e6653e4ee993a5cb24a836ae45 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/06b3a1e6653e4ee993a5cb24a836ae45 2024-12-05T12:16:20,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/4535cb4a5eba425485cfabfdd021b2bd to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/4535cb4a5eba425485cfabfdd021b2bd 2024-12-05T12:16:20,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d5945734c237460faf0210af76f4775d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d5945734c237460faf0210af76f4775d 2024-12-05T12:16:20,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d4bd93353f914b57a5148cfe7850c66f to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/d4bd93353f914b57a5148cfe7850c66f 2024-12-05T12:16:20,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1f94d4d941514f559c162b608565eaaf to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/1f94d4d941514f559c162b608565eaaf 2024-12-05T12:16:20,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/8b2b7d36383541cd861a0d3f40a2e0b0 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/8b2b7d36383541cd861a0d3f40a2e0b0 2024-12-05T12:16:20,813 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/recovered.edits/313.seqid, newMaxSeqId=313, maxSeqId=4 2024-12-05T12:16:20,814 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. 2024-12-05T12:16:20,814 DEBUG [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 2e5f943f8689f12e595bd09f366346aa: Waiting for close lock at 1733400978577Running coprocessor pre-close hooks at 1733400978577Disabling compacts and flushes for region at 1733400978577Disabling writes for close at 1733400979496 (+919 ms)Obtaining lock to block concurrent updates at 1733400979496Preparing flush snapshotting stores in 2e5f943f8689f12e595bd09f366346aa at 1733400979496Finished memstore snapshotting TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733400979496Flushing stores of TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa. at 1733400979497 (+1 ms)Flushing 2e5f943f8689f12e595bd09f366346aa/A: creating writer at 1733400979497Flushing mob file 2e5f943f8689f12e595bd09f366346aa/A: appending metadata at 1733400979500 (+3 ms)Flushing mob file 2e5f943f8689f12e595bd09f366346aa/A: closing flushed file at 1733400979500Flushing 2e5f943f8689f12e595bd09f366346aa/A: appending metadata at 1733400979915 (+415 ms)Flushing 2e5f943f8689f12e595bd09f366346aa/A: closing flushed file at 1733400979915Flushing 2e5f943f8689f12e595bd09f366346aa/B: creating writer at 1733400980330 (+415 ms)Flushing 2e5f943f8689f12e595bd09f366346aa/B: appending metadata at 1733400980333 (+3 ms)Flushing 2e5f943f8689f12e595bd09f366346aa/B: closing flushed file at 1733400980333Flushing 2e5f943f8689f12e595bd09f366346aa/C: creating writer at 1733400980342 (+9 ms)Flushing 2e5f943f8689f12e595bd09f366346aa/C: appending metadata at 1733400980345 (+3 ms)Flushing 2e5f943f8689f12e595bd09f366346aa/C: closing flushed file at 1733400980345Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@174e624a: reopening flushed file at 1733400980757 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e22862f: reopening flushed file at 1733400980761 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42d004d3: reopening flushed file at 1733400980766 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 2e5f943f8689f12e595bd09f366346aa in 1274ms, sequenceid=310, compaction requested=true at 1733400980770 (+4 ms)Writing region close event to WAL at 1733400980811 (+41 ms)Running coprocessor post-close hooks at 1733400980813 (+2 ms)Closed at 1733400980814 (+1 ms) 2024-12-05T12:16:20,815 INFO [RS_CLOSE_REGION-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:20,816 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=2e5f943f8689f12e595bd09f366346aa, regionState=CLOSED 2024-12-05T12:16:20,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 because future has completed 2024-12-05T12:16:20,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-05T12:16:20,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 2e5f943f8689f12e595bd09f366346aa, server=9ea9abad0e9b,44759,1733400796278 in 2.3930 sec 2024-12-05T12:16:20,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-05T12:16:20,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e5f943f8689f12e595bd09f366346aa, UNASSIGN in 2.3970 sec 2024-12-05T12:16:20,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-05T12:16:20,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4000 sec 2024-12-05T12:16:20,822 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400980822"}]},"ts":"1733400980822"} 2024-12-05T12:16:20,823 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T12:16:20,823 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T12:16:20,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4080 sec 2024-12-05T12:16:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T12:16:22,574 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:16:22,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T12:16:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:16:22,578 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:16:22,579 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:16:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-05T12:16:22,582 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,584 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C, FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/recovered.edits] 2024-12-05T12:16:22,586 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0d0c5c68aded4e1280f373fdc0ea4167 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/0d0c5c68aded4e1280f373fdc0ea4167 2024-12-05T12:16:22,586 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/3468f4a311404704b9bcd7e434512772 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/3468f4a311404704b9bcd7e434512772 2024-12-05T12:16:22,587 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/82632152efdc4435b35f34487eae318b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/A/82632152efdc4435b35f34487eae318b 2024-12-05T12:16:22,589 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/91d748aa37234ccea7673e6538144b1d to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/91d748aa37234ccea7673e6538144b1d 2024-12-05T12:16:22,589 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9d30493753794189a6ecb7af8fc7b58e to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9d30493753794189a6ecb7af8fc7b58e 2024-12-05T12:16:22,590 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9d7523561ff8436a9e9fbee2bfee3b5b to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/B/9d7523561ff8436a9e9fbee2bfee3b5b 2024-12-05T12:16:22,591 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/01ee0300e74941e1a2684b1bbd77e8e7 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/01ee0300e74941e1a2684b1bbd77e8e7 2024-12-05T12:16:22,592 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/3a4c2304c76d4e5a9668e49a9be77f11 to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/3a4c2304c76d4e5a9668e49a9be77f11 2024-12-05T12:16:22,592 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/b59f53783bf7462b8a6f2566e020f1cf to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/C/b59f53783bf7462b8a6f2566e020f1cf 2024-12-05T12:16:22,594 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/recovered.edits/313.seqid to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa/recovered.edits/313.seqid 2024-12-05T12:16:22,594 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/default/TestAcidGuarantees/2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,594 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T12:16:22,595 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T12:16:22,595 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T12:16:22,597 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050984c953bd924a4484e9e4ac8fe3364b_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050984c953bd924a4484e9e4ac8fe3364b_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,597 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120509f45aef917a49eeba1eab8ac056f5aa_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120509f45aef917a49eeba1eab8ac056f5aa_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,598 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050fab060729ce4016bf6bdd3ab1635c0c_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050fab060729ce4016bf6bdd3ab1635c0c_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,599 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120510d9cf71d7704b2ea2a4af20503c6cc5_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120510d9cf71d7704b2ea2a4af20503c6cc5_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,600 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120522801ab32e7049ae9e3aa2458638c947_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120522801ab32e7049ae9e3aa2458638c947_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,600 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120539afa32bb4a9484f91bd8b4b177ca34b_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120539afa32bb4a9484f91bd8b4b177ca34b_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,601 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053c889d43598b4e969444c1e43b788b18_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053c889d43598b4e969444c1e43b788b18_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,601 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120543720829b3ac45f5b7f3164d9825480a_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120543720829b3ac45f5b7f3164d9825480a_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,602 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055c1c0d0817834a97aa152ba85df8ab44_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055c1c0d0817834a97aa152ba85df8ab44_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,603 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205876b078798454cf5836bdb0bb564fb38_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205876b078798454cf5836bdb0bb564fb38_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,604 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059b7903e3c43740a8a7fd18ecceac2b40_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059b7903e3c43740a8a7fd18ecceac2b40_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,604 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b727d36808794913adb11e8adac8e0fc_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b727d36808794913adb11e8adac8e0fc_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,605 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c707e9c5641e484c8b7d733bd5685b1e_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c707e9c5641e484c8b7d733bd5685b1e_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,606 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cb26e1ae28d3485f843fc23e40e04611_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cb26e1ae28d3485f843fc23e40e04611_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,606 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d241850caa0846348c1afd17c0d6f2b0_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d241850caa0846348c1afd17c0d6f2b0_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,607 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ec4e368bac704e0b8e423dfbeb02f833_2e5f943f8689f12e595bd09f366346aa to hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ec4e368bac704e0b8e423dfbeb02f833_2e5f943f8689f12e595bd09f366346aa 2024-12-05T12:16:22,607 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T12:16:22,609 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:16:22,612 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T12:16:22,613 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T12:16:22,614 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:16:22,614 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T12:16:22,614 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733400982614"}]},"ts":"9223372036854775807"} 2024-12-05T12:16:22,616 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T12:16:22,616 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2e5f943f8689f12e595bd09f366346aa, NAME => 'TestAcidGuarantees,,1733400953569.2e5f943f8689f12e595bd09f366346aa.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T12:16:22,616 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T12:16:22,616 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733400982616"}]},"ts":"9223372036854775807"} 2024-12-05T12:16:22,618 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T12:16:22,618 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T12:16:22,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-12-05T12:16:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34103 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-05T12:16:22,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T12:16:22,683 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T12:16:22,691 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249 (was 250), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=413 (was 467), ProcessCount=11 (was 11), AvailableMemoryMB=3739 (was 3791) 2024-12-05T12:16:22,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T12:16:22,692 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:16:22,692 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:16:22,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:22,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:22,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:16:22,692 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T12:16:22,692 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1519452659, stopped=false 2024-12-05T12:16:22,693 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9ea9abad0e9b,34103,1733400795471 2024-12-05T12:16:22,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T12:16:22,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T12:16:22,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:16:22,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:16:22,724 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T12:16:22,725 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:16:22,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:16:22,726 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:16:22,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:22,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:16:22,727 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ea9abad0e9b,44759,1733400796278' ***** 2024-12-05T12:16:22,727 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T12:16:22,730 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T12:16:22,730 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T12:16:22,730 INFO [RS:0;9ea9abad0e9b:44759 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T12:16:22,731 INFO [RS:0;9ea9abad0e9b:44759 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T12:16:22,731 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(959): stopping server 9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:22,731 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T12:16:22,731 INFO [RS:0;9ea9abad0e9b:44759 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9ea9abad0e9b:44759. 2024-12-05T12:16:22,731 DEBUG [RS:0;9ea9abad0e9b:44759 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:16:22,731 DEBUG [RS:0;9ea9abad0e9b:44759 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:22,731 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T12:16:22,731 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T12:16:22,731 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T12:16:22,731 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T12:16:22,732 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T12:16:22,732 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-05T12:16:22,732 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T12:16:22,732 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T12:16:22,732 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T12:16:22,732 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T12:16:22,732 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T12:16:22,732 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T12:16:22,733 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-05T12:16:22,755 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/info/1824c5a5af264413b59b2f3e91366262 is 95, key is TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a./info:/1733400835851/DeleteFamily/seqid=0 2024-12-05T12:16:22,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742426_1602 (size=6514) 2024-12-05T12:16:22,793 INFO [regionserver/9ea9abad0e9b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:16:22,933 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T12:16:23,133 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T12:16:23,160 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/info/1824c5a5af264413b59b2f3e91366262 2024-12-05T12:16:23,184 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/ns/8a93f1d2b5834359ba9ed17bd42fa6af is 93, key is TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a./ns:/1733400835797/DeleteFamily/seqid=0 2024-12-05T12:16:23,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742427_1603 (size=5906) 2024-12-05T12:16:23,333 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T12:16:23,469 INFO [regionserver/9ea9abad0e9b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T12:16:23,469 INFO [regionserver/9ea9abad0e9b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T12:16:23,534 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T12:16:23,589 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/ns/8a93f1d2b5834359ba9ed17bd42fa6af 2024-12-05T12:16:23,612 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/rep_barrier/0191714517b7495f83fded4ab9786986 is 102, key is TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a./rep_barrier:/1733400835797/DeleteFamily/seqid=0 2024-12-05T12:16:23,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742428_1604 (size=6025) 2024-12-05T12:16:23,734 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T12:16:23,934 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T12:16:24,017 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/rep_barrier/0191714517b7495f83fded4ab9786986 2024-12-05T12:16:24,041 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/table/ee966963432947e9881bd9702b777207 is 96, key is TestAcidGuarantees,,1733400801014.9c2bbad557c305bbacb16e1dfd91bc9a./table:/1733400835797/DeleteFamily/seqid=0 2024-12-05T12:16:24,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742429_1605 (size=5936) 2024-12-05T12:16:24,135 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T12:16:24,335 DEBUG [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T12:16:24,445 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/table/ee966963432947e9881bd9702b777207 2024-12-05T12:16:24,455 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/info/1824c5a5af264413b59b2f3e91366262 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/info/1824c5a5af264413b59b2f3e91366262 2024-12-05T12:16:24,458 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/info/1824c5a5af264413b59b2f3e91366262, entries=12, sequenceid=90, filesize=6.4 K 2024-12-05T12:16:24,459 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/ns/8a93f1d2b5834359ba9ed17bd42fa6af as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/ns/8a93f1d2b5834359ba9ed17bd42fa6af 2024-12-05T12:16:24,461 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/ns/8a93f1d2b5834359ba9ed17bd42fa6af, entries=8, sequenceid=90, filesize=5.8 K 2024-12-05T12:16:24,462 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/rep_barrier/0191714517b7495f83fded4ab9786986 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/rep_barrier/0191714517b7495f83fded4ab9786986 2024-12-05T12:16:24,465 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/rep_barrier/0191714517b7495f83fded4ab9786986, entries=6, sequenceid=90, filesize=5.9 K 2024-12-05T12:16:24,465 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/.tmp/table/ee966963432947e9881bd9702b777207 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/table/ee966963432947e9881bd9702b777207 2024-12-05T12:16:24,468 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/table/ee966963432947e9881bd9702b777207, entries=7, sequenceid=90, filesize=5.8 K 2024-12-05T12:16:24,481 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1749ms, sequenceid=90, compaction requested=false 2024-12-05T12:16:24,484 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-05T12:16:24,485 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T12:16:24,485 INFO [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T12:16:24,485 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733400982732Running coprocessor pre-close hooks at 1733400982732Disabling compacts and flushes for region at 1733400982732Disabling writes for close at 1733400982732Obtaining lock to block concurrent updates at 1733400982733 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733400982733Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733400982733Flushing stores of hbase:meta,,1.1588230740 at 1733400982734 (+1 ms)Flushing 1588230740/info: creating writer at 1733400982734Flushing 1588230740/info: appending metadata at 1733400982755 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733400982755Flushing 1588230740/ns: creating writer at 1733400983172 (+417 ms)Flushing 1588230740/ns: appending metadata at 1733400983184 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733400983184Flushing 1588230740/rep_barrier: creating writer at 1733400983596 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733400983612 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733400983612Flushing 1588230740/table: creating writer at 1733400984024 (+412 ms)Flushing 1588230740/table: appending metadata at 1733400984041 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733400984041Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b55a2e6: reopening flushed file at 1733400984454 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55d5270d: reopening flushed file at 1733400984458 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b98bb67: reopening flushed file at 1733400984462 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e65e47a: reopening flushed file at 1733400984465 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1749ms, sequenceid=90, compaction requested=false at 1733400984481 (+16 ms)Writing region close event to WAL at 1733400984482 (+1 ms)Running coprocessor post-close hooks at 1733400984485 (+3 ms)Closed at 1733400984485 2024-12-05T12:16:24,485 DEBUG [RS_CLOSE_META-regionserver/9ea9abad0e9b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T12:16:24,536 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(976): stopping server 9ea9abad0e9b,44759,1733400796278; all regions closed. 2024-12-05T12:16:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741834_1010 (size=25344) 2024-12-05T12:16:24,546 DEBUG [RS:0;9ea9abad0e9b:44759 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/oldWALs 2024-12-05T12:16:24,547 INFO [RS:0;9ea9abad0e9b:44759 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ea9abad0e9b%2C44759%2C1733400796278.meta:.meta(num 1733400800513) 2024-12-05T12:16:24,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741832_1008 (size=16508737) 2024-12-05T12:16:24,551 DEBUG [RS:0;9ea9abad0e9b:44759 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/oldWALs 2024-12-05T12:16:24,551 INFO [RS:0;9ea9abad0e9b:44759 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ea9abad0e9b%2C44759%2C1733400796278:(num 1733400799532) 2024-12-05T12:16:24,551 DEBUG [RS:0;9ea9abad0e9b:44759 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:16:24,551 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:16:24,551 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T12:16:24,552 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.ChoreService(370): Chore service for: regionserver/9ea9abad0e9b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T12:16:24,552 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T12:16:24,552 INFO [regionserver/9ea9abad0e9b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T12:16:24,553 INFO [RS:0;9ea9abad0e9b:44759 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44759 2024-12-05T12:16:24,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T12:16:24,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ea9abad0e9b,44759,1733400796278 2024-12-05T12:16:24,571 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T12:16:24,572 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ea9abad0e9b,44759,1733400796278] 2024-12-05T12:16:24,590 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ea9abad0e9b,44759,1733400796278 already deleted, retry=false 2024-12-05T12:16:24,591 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9ea9abad0e9b,44759,1733400796278 expired; onlineServers=0 2024-12-05T12:16:24,591 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9ea9abad0e9b,34103,1733400795471' ***** 2024-12-05T12:16:24,591 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T12:16:24,591 INFO [M:0;9ea9abad0e9b:34103 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T12:16:24,592 INFO [M:0;9ea9abad0e9b:34103 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T12:16:24,592 DEBUG [M:0;9ea9abad0e9b:34103 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T12:16:24,592 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T12:16:24,592 DEBUG [M:0;9ea9abad0e9b:34103 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T12:16:24,592 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster-HFileCleaner.large.0-1733400799200 {}] cleaner.HFileCleaner(306): Exit Thread[master/9ea9abad0e9b:0:becomeActiveMaster-HFileCleaner.large.0-1733400799200,5,FailOnTimeoutGroup] 2024-12-05T12:16:24,592 DEBUG [master/9ea9abad0e9b:0:becomeActiveMaster-HFileCleaner.small.0-1733400799201 {}] cleaner.HFileCleaner(306): Exit Thread[master/9ea9abad0e9b:0:becomeActiveMaster-HFileCleaner.small.0-1733400799201,5,FailOnTimeoutGroup] 2024-12-05T12:16:24,593 INFO [M:0;9ea9abad0e9b:34103 {}] hbase.ChoreService(370): Chore service for: master/9ea9abad0e9b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T12:16:24,593 INFO [M:0;9ea9abad0e9b:34103 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T12:16:24,593 DEBUG [M:0;9ea9abad0e9b:34103 {}] master.HMaster(1795): Stopping service threads 2024-12-05T12:16:24,593 INFO [M:0;9ea9abad0e9b:34103 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T12:16:24,593 INFO [M:0;9ea9abad0e9b:34103 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T12:16:24,594 ERROR [M:0;9ea9abad0e9b:34103 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-05T12:16:24,595 INFO [M:0;9ea9abad0e9b:34103 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T12:16:24,595 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T12:16:24,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T12:16:24,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:16:24,604 DEBUG [M:0;9ea9abad0e9b:34103 {}] zookeeper.ZKUtil(347): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T12:16:24,604 WARN [M:0;9ea9abad0e9b:34103 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T12:16:24,605 INFO [M:0;9ea9abad0e9b:34103 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/.lastflushedseqids 2024-12-05T12:16:24,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742430_1606 (size=72) 2024-12-05T12:16:24,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:16:24,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44759-0x101a696db740001, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:16:24,684 INFO [RS:0;9ea9abad0e9b:44759 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T12:16:24,684 INFO [RS:0;9ea9abad0e9b:44759 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ea9abad0e9b,44759,1733400796278; zookeeper connection closed. 2024-12-05T12:16:24,685 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@36714918 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@36714918 2024-12-05T12:16:24,686 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T12:16:25,021 INFO [M:0;9ea9abad0e9b:34103 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T12:16:25,022 INFO [M:0;9ea9abad0e9b:34103 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T12:16:25,022 DEBUG [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T12:16:25,022 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:16:25,023 DEBUG [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:16:25,023 DEBUG [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T12:16:25,023 DEBUG [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:16:25,023 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=783.50 KB heapSize=960.95 KB 2024-12-05T12:16:25,044 DEBUG [M:0;9ea9abad0e9b:34103 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/981a38520ea647c895a598286ae7e18b is 82, key is hbase:meta,,1/info:regioninfo/1733400800595/Put/seqid=0 2024-12-05T12:16:25,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742431_1607 (size=5672) 2024-12-05T12:16:25,449 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2198 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/981a38520ea647c895a598286ae7e18b 2024-12-05T12:16:25,478 DEBUG [M:0;9ea9abad0e9b:34103 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9fc9fd425fb24503bfcc16e18b1a573b is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1733400956663/Put/seqid=0 2024-12-05T12:16:25,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742432_1608 (size=44650) 2024-12-05T12:16:25,882 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=782.94 KB at sequenceid=2198 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9fc9fd425fb24503bfcc16e18b1a573b 2024-12-05T12:16:25,887 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9fc9fd425fb24503bfcc16e18b1a573b 2024-12-05T12:16:25,904 DEBUG [M:0;9ea9abad0e9b:34103 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e115083d1d8941928d56163b06664680 is 69, key is 9ea9abad0e9b,44759,1733400796278/rs:state/1733400799310/Put/seqid=0 2024-12-05T12:16:25,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073742433_1609 (size=5156) 2024-12-05T12:16:26,308 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2198 (bloomFilter=true), to=hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e115083d1d8941928d56163b06664680 2024-12-05T12:16:26,313 DEBUG [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/981a38520ea647c895a598286ae7e18b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/981a38520ea647c895a598286ae7e18b 2024-12-05T12:16:26,316 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/981a38520ea647c895a598286ae7e18b, entries=8, sequenceid=2198, filesize=5.5 K 2024-12-05T12:16:26,317 DEBUG [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9fc9fd425fb24503bfcc16e18b1a573b as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9fc9fd425fb24503bfcc16e18b1a573b 2024-12-05T12:16:26,320 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9fc9fd425fb24503bfcc16e18b1a573b 2024-12-05T12:16:26,320 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9fc9fd425fb24503bfcc16e18b1a573b, entries=166, sequenceid=2198, filesize=43.6 K 2024-12-05T12:16:26,321 DEBUG [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e115083d1d8941928d56163b06664680 as hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e115083d1d8941928d56163b06664680 2024-12-05T12:16:26,324 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46811/user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e115083d1d8941928d56163b06664680, entries=1, sequenceid=2198, filesize=5.0 K 2024-12-05T12:16:26,325 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegion(3140): Finished flush of dataSize ~783.50 KB/802300, heapSize ~960.65 KB/983704, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1302ms, sequenceid=2198, compaction requested=false 2024-12-05T12:16:26,327 INFO [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:16:26,327 DEBUG [M:0;9ea9abad0e9b:34103 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733400985022Disabling compacts and flushes for region at 1733400985022Disabling writes for close at 1733400985023 (+1 ms)Obtaining lock to block concurrent updates at 1733400985023Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733400985023Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=802300, getHeapSize=983944, getOffHeapSize=0, getCellsCount=2389 at 1733400985024 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733400985025 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733400985025Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733400985044 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733400985044Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733400985459 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733400985478 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733400985478Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733400985887 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733400985904 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733400985904Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38545527: reopening flushed file at 1733400986312 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c8d3663: reopening flushed file at 1733400986316 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56d3654d: reopening flushed file at 1733400986320 (+4 ms)Finished flush of dataSize ~783.50 KB/802300, heapSize ~960.65 KB/983704, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1302ms, sequenceid=2198, compaction requested=false at 1733400986325 (+5 ms)Writing region close event to WAL at 1733400986326 (+1 ms)Closed at 1733400986326 2024-12-05T12:16:26,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32993 is added to blk_1073741830_1006 (size=946353) 2024-12-05T12:16:26,329 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/15650de1-446c-2138-d0f2-47531d1d3c4e/MasterData/WALs/9ea9abad0e9b,34103,1733400795471/9ea9abad0e9b%2C34103%2C1733400795471.1733400798421 not finished, retry = 0 2024-12-05T12:16:26,333 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T12:16:26,354 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T12:16:26,354 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T12:16:26,354 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T12:16:26,354 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T12:16:26,430 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T12:16:26,430 INFO [M:0;9ea9abad0e9b:34103 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T12:16:26,430 INFO [M:0;9ea9abad0e9b:34103 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34103 2024-12-05T12:16:26,430 INFO [M:0;9ea9abad0e9b:34103 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T12:16:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:16:26,540 INFO [M:0;9ea9abad0e9b:34103 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T12:16:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34103-0x101a696db740000, quorum=127.0.0.1:59678, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:16:26,545 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T12:16:26,547 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T12:16:26,547 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T12:16:26,547 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T12:16:26,547 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/hadoop.log.dir/,STOPPED} 2024-12-05T12:16:26,550 WARN [BP-1803855376-172.17.0.3-1733400791885 heartbeating to localhost/127.0.0.1:46811 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T12:16:26,550 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T12:16:26,550 WARN [BP-1803855376-172.17.0.3-1733400791885 heartbeating to localhost/127.0.0.1:46811 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1803855376-172.17.0.3-1733400791885 (Datanode Uuid 8d221f7e-d96d-4699-8801-d839600c5300) service to localhost/127.0.0.1:46811 2024-12-05T12:16:26,550 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T12:16:26,553 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/cluster_9f95a8c6-304c-dd62-ec56-918ba890d196/data/data1/current/BP-1803855376-172.17.0.3-1733400791885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T12:16:26,554 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/cluster_9f95a8c6-304c-dd62-ec56-918ba890d196/data/data2/current/BP-1803855376-172.17.0.3-1733400791885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T12:16:26,555 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T12:16:26,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T12:16:26,561 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T12:16:26,561 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T12:16:26,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T12:16:26,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b9a4e03-85d9-0946-1e4d-be827fcec3d3/hadoop.log.dir/,STOPPED} 2024-12-05T12:16:26,575 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T12:16:26,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down